diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0bea5de1df1..e78d7a18c85 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -57,7 +57,7 @@ jobs: - name: Use benchstat for comparison run: | export PATH=$PATH:$(go env GOPATH)/bin - go install golang.org/x/perf/cmd/benchstat@latest + go install golang.org/x/perf/cmd/benchstat@91a04616dc65ba76dbe9e5cf746b923b1402d303 echo "BENCHSTAT<> $GITHUB_ENV echo "$(benchstat -html -sort delta old.txt new.txt | sed '//d' | sed 's/<!doctype html>//g')" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0bca5d4aba..9b977950c97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: cache: true - name: Run tests (${{ matrix.targets.name }}) if: github.actor != 'bors[bot]' - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v2 with: timeout_minutes: 25 max_attempts: 3 @@ -128,7 +128,7 @@ jobs: # RACE_DETECTOR: 1 - name: Run tests (Bors) if: github.actor == 'bors[bot]' - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v2 with: timeout_minutes: 25 max_attempts: 3 @@ -180,7 +180,7 @@ jobs: VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }} - name: Run tests (Bors) if: github.actor == 'bors[bot]' - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v2 with: timeout_minutes: 25 max_attempts: ${{ matrix.retries }} @@ -231,7 +231,7 @@ jobs: # RACE_DETECTOR: 1 - name: Run tests (Bors) if: github.actor == 'bors[bot]' - uses: nick-invision/retry@v2 + uses: nick-fields/retry@v2 with: timeout_minutes: 15 max_attempts: 2 diff --git a/LicensingApproach.md b/LicensingApproach.md new file mode 100644 index 00000000000..2360e32f793 --- /dev/null +++ b/LicensingApproach.md @@ -0,0 +1,31 @@ +# Flow's Approach to Licensing + +The founding team for Flow suggests that all open-source software built on or for Flow should adhere to the "[Coherent Open Source Licensing](https://licenseuse.org)" philosophy. +To that end, we encourage the use of one of the following three licenses, as applicable by the use case (order from most permissive, to most restrictive): + +- Apache 2.0 +- LGPL 3 +- Affero GPL 3 + +These licenses are all approved by both the Free Software Foundation and the Open Source Initiative and contain clauses regarding software patenting to avoid hidden patent concerns. Additionally, they are all compatible with each other meaning there is no conflict using them together in any project (provided the project as a whole meets the requirements of the most restrictive of the licenses included). + + +### Apache 2.0 + +Apache 2.0 is the most permissive, and allows reuse - with customization - in proprietary software. We recommend that this should be the default for any code that has significant potential use for off-chain tools and/or applications. In particular, all sample code (including smart contracts), SDKs, utility libraries and tools should default to Apache 2.0. + + +### LGPL 3 + +LGPL 3 allows use in proprietary software, provided that any customizations to the licensed code are shared. We considered LGPL for Cadence to avoid a proprietary fork of the language, but decided to use the more permissive Apache 2.0 license to emphasize that we hope Cadence can be adopted outside of Flow. LGPL can be used for other parts of Flow where necessary (e.g. if the component is itself a modification of other LGPL-licensed code). + + +### Affero GPL 3 + +Affero GPL 3 (AGPL) should be used for all code that exists primarily to power node software. In particular, the core consensus and computation layers should be provided via AGPL where possible. We chose this license because we interpret the clauses in the AGPL regarding "public use [...] on a publicly accessible server" as covering the operation of a node in the public Flow network. By requiring all modifications to the core node software to be made publicly available, we increase the security of the network (allowing any custom modifications to undergo public scrutiny), while also ensuring that improvements to the core infrastructure benefit all participants in the network. + + +### Using the Unlicense + +For small snippets of sample code, we can provide access via the Unlicense. +Broadly speaking, Apache 2.0 gives essentially the same rights to the receiver of the code as Unlicense, except that Apache protects the author(s) a bit more robustly, and protects the receiver against patent lawsuits. But, some people aren't aware of this, and using the "Unlicense" makes it very clear to anyone that the original authors aren't claiming ownership of derivative works. For any "complex" code that we want to be free, we should use Apache. For simple code snippets and examples, the Unlicense will probably make casual/inexperienced developers feel more comfortable. diff --git a/Makefile b/Makefile index 02e892ff33f..c484cd1be54 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,10 @@ cmd/collection/collection: cmd/util/util: go build -o cmd/util/util --tags relic cmd/util/main.go +.PHONY: update-core-contracts-version +update-core-contracts-version: + ./scripts/update-core-contracts.sh $(CC_VERSION) + ############################################################################################ # CAUTION: DO NOT MODIFY THESE TARGETS! DOING SO WILL BREAK THE FLAKY TEST MONITOR @@ -151,6 +155,7 @@ generate-mocks: install-mock-generators mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock" mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock" mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock" + mockery --name '.*' --dir=engine/collection --case=underscore --output="./engine/collection/mock" --outpkg="mock" mockery --name '.*' --dir=engine/consensus --case=underscore --output="./engine/consensus/mock" --outpkg="mock" mockery --name '.*' --dir=engine/consensus/approvals --case=underscore --output="./engine/consensus/approvals/mock" --outpkg="mock" rm -rf ./fvm/environment/mock diff --git a/access/handler.go b/access/handler.go index c78f7eace48..914fd2a805d 100644 --- a/access/handler.go +++ b/access/handler.go @@ -492,7 +492,7 @@ func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access. func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flow.BlockStatus) (*access.BlockResponse, error) { signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(block.Header) if err != nil { - return nil, err + return nil, err // the block was retrieved from local storage - so no errors are expected } var msg *entities.Block @@ -513,7 +513,7 @@ func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flo func (h *Handler) blockHeaderResponse(header *flow.Header, status flow.BlockStatus) (*access.BlockHeaderResponse, error) { signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(header) if err != nil { - return nil, err + return nil, err // the block was retrieved from local storage - so no errors are expected } msg, err := convert.BlockHeaderToMessage(header, signerIDs) diff --git a/admin/command_runner.go b/admin/command_runner.go index 3ca0a1aa20f..3de41fb73ae 100644 --- a/admin/command_runner.go +++ b/admin/command_runner.go @@ -324,3 +324,7 @@ func (r *CommandRunner) runCommand(ctx context.Context, command string, data int return handleResult, nil } + +func (r *CommandRunner) GrpcAddress() string { + return r.grpcAddress +} diff --git a/apiproxy/access_api_proxy.go b/apiproxy/access_api_proxy.go new file mode 100644 index 00000000000..8e0b781af5e --- /dev/null +++ b/apiproxy/access_api_proxy.go @@ -0,0 +1,468 @@ +package apiproxy + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + "github.com/onflow/flow/protobuf/go/flow/access" + + "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/grpcutils" +) + +// NewFlowAccessAPIRouter creates a backend access API that forwards some requests to an upstream node. +// It is used by Observer services, Blockchain Data Service, etc. +// Make sure that this is just for observation and not a staked participant in the flow network. +// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. +func NewFlowAccessAPIRouter(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) (*FlowAccessAPIRouter, error) { + ret := &FlowAccessAPIRouter{} + err := ret.upstream.setFlowAccessAPI(accessNodeAddressAndPort, timeout) + if err != nil { + return nil, err + } + return ret, nil +} + +// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. +// It is used by Observer services, Blockchain Data Service, etc. +// Make sure that this is just for observation and not a staked participant in the flow network. +// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. +func (ret *FlowAccessAPIForwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { + ret.timeout = timeout + ret.ids = accessNodeAddressAndPort + ret.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) + ret.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) + for i, identity := range accessNodeAddressAndPort { + // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that + // we can refresh the API on connection loss + ret.ids[i] = identity + + // We fail on any single error on startup, so that + // we identify bootstrapping errors early + err := ret.reconnectingClient(i) + if err != nil { + return err + } + } + + ret.roundRobin = 0 + return nil +} + +// FlowAccessAPIRouter is a structure that represents the routing proxy algorithm. +// It splits requests between a local and a remote API service. +type FlowAccessAPIRouter struct { + access.AccessAPIServer + upstream FlowAccessAPIForwarder +} + +// SetLocalAPI sets the local backend that responds to block related calls +// Everything else is forwarded to a selected upstream node +func (h *FlowAccessAPIRouter) SetLocalAPI(local access.AccessAPIServer) { + h.AccessAPIServer = local +} + +// reconnectingClient returns an active client, or +// creates one, if the last one is not ready anymore. +func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { + timeout := h.timeout + + if h.connections[i] == nil || h.connections[i].GetState() != connectivity.Ready { + identity := h.ids[i] + var connection *grpc.ClientConn + var err error + if identity.NetworkPubKey == nil { + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithInsecure(), //nolint:staticcheck + backend.WithClientUnaryInterceptor(timeout)) + if err != nil { + return err + } + } else { + tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) + if err != nil { + return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) + } + + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + backend.WithClientUnaryInterceptor(timeout)) + if err != nil { + return fmt.Errorf("cannot connect to %s %w", identity.Address, err) + } + } + connection.Connect() + time.Sleep(1 * time.Second) + state := connection.GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + return fmt.Errorf("%v", state) + } + h.connections[i] = connection + h.upstream[i] = access.NewAccessAPIClient(connection) + } + + return nil +} + +// faultTolerantClient implements an upstream connection that reconnects on errors +// a reasonable amount of time. +func (h *FlowAccessAPIForwarder) faultTolerantClient() (access.AccessAPIClient, error) { + if h.upstream == nil || len(h.upstream) == 0 { + return nil, status.Errorf(codes.Unimplemented, "method not implemented") + } + + // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. + // A bigger number is problematic due to the DNS resolve and connection times, + // plus the need to log and debug each individual connection failure. + // + // This reasoning eliminates the need of making this parameter configurable. + // The logic works rolling over a single connection as well making clean code. + const retryMax = 3 + + h.lock.Lock() + defer h.lock.Unlock() + + var err error + for i := 0; i < retryMax; i++ { + h.roundRobin++ + h.roundRobin = h.roundRobin % len(h.upstream) + err = h.reconnectingClient(h.roundRobin) + if err != nil { + continue + } + state := h.connections[h.roundRobin].GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + continue + } + return h.upstream[h.roundRobin], nil + } + + return nil, status.Errorf(codes.Unavailable, err.Error()) +} + +// Ping pings the service. It is special in the sense that it responds successful, +// only if all underlying services are ready. +func (h *FlowAccessAPIRouter) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { + return h.AccessAPIServer.Ping(context, req) +} + +func (h *FlowAccessAPIRouter) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { + return h.AccessAPIServer.GetLatestBlockHeader(context, req) +} + +func (h *FlowAccessAPIRouter) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { + return h.AccessAPIServer.GetBlockHeaderByID(context, req) +} + +func (h *FlowAccessAPIRouter) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { + return h.AccessAPIServer.GetBlockHeaderByHeight(context, req) +} + +func (h *FlowAccessAPIRouter) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { + return h.AccessAPIServer.GetLatestBlock(context, req) +} + +func (h *FlowAccessAPIRouter) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { + return h.AccessAPIServer.GetBlockByID(context, req) +} + +func (h *FlowAccessAPIRouter) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { + return h.AccessAPIServer.GetBlockByHeight(context, req) +} + +func (h *FlowAccessAPIRouter) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { + return h.AccessAPIServer.GetCollectionByID(context, req) +} + +func (h *FlowAccessAPIRouter) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { + return h.upstream.SendTransaction(context, req) +} + +func (h *FlowAccessAPIRouter) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { + return h.upstream.GetTransaction(context, req) +} + +func (h *FlowAccessAPIRouter) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { + return h.upstream.GetTransactionResult(context, req) +} + +func (h *FlowAccessAPIRouter) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { + return h.upstream.GetTransactionResultByIndex(context, req) +} + +func (h *FlowAccessAPIRouter) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { + return h.upstream.GetAccount(context, req) +} + +func (h *FlowAccessAPIRouter) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { + return h.upstream.GetAccountAtLatestBlock(context, req) +} + +func (h *FlowAccessAPIRouter) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { + return h.upstream.GetAccountAtBlockHeight(context, req) +} + +func (h *FlowAccessAPIRouter) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { + return h.upstream.ExecuteScriptAtLatestBlock(context, req) +} + +func (h *FlowAccessAPIRouter) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { + return h.upstream.ExecuteScriptAtBlockID(context, req) +} + +func (h *FlowAccessAPIRouter) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { + return h.upstream.ExecuteScriptAtBlockHeight(context, req) +} + +func (h *FlowAccessAPIRouter) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { + return h.upstream.GetEventsForHeightRange(context, req) +} + +func (h *FlowAccessAPIRouter) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { + return h.upstream.GetEventsForBlockIDs(context, req) +} + +func (h *FlowAccessAPIRouter) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { + return h.AccessAPIServer.GetNetworkParameters(context, req) +} + +func (h *FlowAccessAPIRouter) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { + return h.AccessAPIServer.GetLatestProtocolStateSnapshot(context, req) +} + +func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { + return h.upstream.GetExecutionResultForBlockID(context, req) +} + +// FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers +type FlowAccessAPIForwarder struct { + lock sync.Mutex + roundRobin int + ids flow.IdentityList + upstream []access.AccessAPIClient + connections []*grpc.ClientConn + timeout time.Duration +} + +// Ping pings the service. It is special in the sense that it responds successful, +// only if all underlying services are ready. +func (h *FlowAccessAPIForwarder) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.Ping(context, req) +} + +func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetLatestBlockHeader(context, req) +} + +func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetBlockHeaderByID(context, req) +} + +func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetBlockHeaderByHeight(context, req) +} + +func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetLatestBlock(context, req) +} + +func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetBlockByID(context, req) +} + +func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetBlockByHeight(context, req) +} + +func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetCollectionByID(context, req) +} + +func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.SendTransaction(context, req) +} + +func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetTransaction(context, req) +} + +func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetTransactionResult(context, req) +} + +func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetTransactionResultByIndex(context, req) +} + +func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetAccount(context, req) +} + +func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetAccountAtLatestBlock(context, req) +} + +func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetAccountAtBlockHeight(context, req) +} + +func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.ExecuteScriptAtLatestBlock(context, req) +} + +func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.ExecuteScriptAtBlockID(context, req) +} + +func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.ExecuteScriptAtBlockHeight(context, req) +} + +func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetEventsForHeightRange(context, req) +} + +func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetEventsForBlockIDs(context, req) +} + +func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetNetworkParameters(context, req) +} + +func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetLatestProtocolStateSnapshot(context, req) +} + +func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetExecutionResultForBlockID(context, req) +} diff --git a/apiproxy/access_api_proxy_test.go b/apiproxy/access_api_proxy_test.go new file mode 100644 index 00000000000..85be5054c09 --- /dev/null +++ b/apiproxy/access_api_proxy_test.go @@ -0,0 +1,272 @@ +package apiproxy + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/onflow/flow/protobuf/go/flow/access" + "google.golang.org/grpc" + grpcinsecure "google.golang.org/grpc/credentials/insecure" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/grpcutils" +) + +// Methodology +// +// We test the proxy and fall-over logic to reach basic coverage. +// +// * Basic coverage means that all conditional checks happen once but only once. +// * We embrace the simplest adequate solution to reduce engineering cost. +// * Any use cases requiring multiple conditionals exercised in a row are considered ignorable due to cost constraints. + +// TestNetE2E tests the basic unix network first +func TestNetE2E(t *testing.T) { + done := make(chan int) + // Bring up 1st upstream server + charlie1, err := makeFlowLite("/tmp/TestProxyE2E1", done) + if err != nil { + t.Fatal(err) + } + // Wait until proxy call passes + err = callFlowLite("/tmp/TestProxyE2E1") + if err != nil { + t.Fatal(err) + } + // Bring up 2nd upstream server + charlie2, err := makeFlowLite("/tmp/TestProxyE2E2", done) + if err != nil { + t.Fatal(err) + } + // Both proxy calls should pass + err = callFlowLite("/tmp/TestProxyE2E1") + if err != nil { + t.Fatal(err) + } + err = callFlowLite("/tmp/TestProxyE2E2") + if err != nil { + t.Fatal(err) + } + // Stop 1st upstream server + _ = charlie1.Close() + // Proxy call falls through + err = callFlowLite("/tmp/TestProxyE2E1") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + // Stop 2nd upstream server + _ = charlie2.Close() + // System errors out on shut down servers + err = callFlowLite("/tmp/TestProxyE2E1") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + err = callFlowLite("/tmp/TestProxyE2E2") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + // wait for all + <-done + <-done +} + +// TestgRPCE2E tests whether gRPC works +func TestGRPCE2E(t *testing.T) { + done := make(chan int) + // Bring up 1st upstream server + charlie1, _, err := newFlowLite("unix", "/tmp/TestProxyE2E1", done) + if err != nil { + t.Fatal(err) + } + // Wait until proxy call passes + err = openFlowLite("/tmp/TestProxyE2E1") + if err != nil { + t.Fatal(err) + } + // Bring up 2nd upstream server + charlie2, _, err := newFlowLite("unix", "/tmp/TestProxyE2E2", done) + if err != nil { + t.Fatal(err) + } + // Both proxy calls should pass + err = openFlowLite("/tmp/TestProxyE2E1") + if err != nil { + t.Fatal(err) + } + err = openFlowLite("/tmp/TestProxyE2E2") + if err != nil { + t.Fatal(err) + } + // Stop 1st upstream server + charlie1.Stop() + // Proxy call falls through + err = openFlowLite("/tmp/TestProxyE2E1") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + // Stop 2nd upstream server + charlie2.Stop() + // System errors out on shut down servers + err = openFlowLite("/tmp/TestProxyE2E1") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + err = openFlowLite("/tmp/TestProxyE2E2") + if err == nil { + t.Fatal(fmt.Errorf("backend still active after close")) + } + // wait for all + <-done + <-done +} + +// TestNewFlowCachedAccessAPIProxy tests the round robin end to end +func TestNewFlowCachedAccessAPIProxy(t *testing.T) { + done := make(chan int) + + // Bring up 1st upstream server + charlie1, _, err := newFlowLite("tcp", "127.0.0.1:11634", done) + if err != nil { + t.Fatal(err) + } + + // Prepare a proxy that fails due to the second connection being idle + l := flow.IdentityList{{Address: "127.0.0.1:11634"}, {Address: "127.0.0.1:11635"}} + c := FlowAccessAPIForwarder{} + err = c.setFlowAccessAPI(l, time.Second) + if err == nil { + t.Fatal(fmt.Errorf("should not start with one connection ready")) + } + + // Bring up 2nd upstream server + charlie2, _, err := newFlowLite("tcp", "127.0.0.1:11635", done) + if err != nil { + t.Fatal(err) + } + + background := context.Background() + + // Prepare a proxy + l = flow.IdentityList{{Address: "127.0.0.1:11634"}, {Address: "127.0.0.1:11635"}} + c = FlowAccessAPIForwarder{} + err = c.setFlowAccessAPI(l, time.Second) + if err != nil { + t.Fatal(err) + } + + // Wait until proxy call passes + _, err = c.Ping(background, &access.PingRequest{}) + if err != nil { + t.Fatal(err) + } + + // Wait until proxy call passes + _, err = c.Ping(background, &access.PingRequest{}) + if err != nil { + t.Fatal(err) + } + + // Wait until proxy call passes + _, err = c.Ping(background, &access.PingRequest{}) + if err != nil { + t.Fatal(err) + } + + charlie1.Stop() + charlie2.Stop() + + // Wait until proxy call fails + _, err = c.Ping(background, &access.PingRequest{}) + if err == nil { + t.Fatal(fmt.Errorf("should fail on no connections")) + } + + <-done + <-done +} + +func makeFlowLite(address string, done chan int) (net.Listener, error) { + l, err := net.Listen("unix", address) + if err != nil { + return nil, err + } + + go func(done chan int) { + for { + c, err := l.Accept() + if err != nil { + break + } + + b := make([]byte, 3) + _, _ = c.Read(b) + _, _ = c.Write(b) + _ = c.Close() + } + done <- 1 + }(done) + return l, err +} + +func callFlowLite(address string) error { + c, err := net.Dial("unix", address) + if err != nil { + return err + } + o := []byte("abc") + _, _ = c.Write(o) + i := make([]byte, 3) + _, _ = c.Read(i) + if string(o) != string(i) { + return fmt.Errorf("no match") + } + _ = c.Close() + _ = MockFlowAccessAPI{} + return err +} + +func newFlowLite(network string, address string, done chan int) (*grpc.Server, *net.Listener, error) { + l, err := net.Listen(network, address) + if err != nil { + return nil, nil, err + } + s := grpc.NewServer() + go func(done chan int) { + access.RegisterAccessAPIServer(s, MockFlowAccessAPI{}) + _ = s.Serve(l) + done <- 1 + }(done) + return s, &l, nil +} + +func openFlowLite(address string) error { + c, err := grpc.Dial( + "unix://"+address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithTransportCredentials(grpcinsecure.NewCredentials())) + if err != nil { + return err + } + a := access.NewAccessAPIClient(c) + + background := context.Background() + + _, err = a.Ping(background, &access.PingRequest{}) + if err != nil { + return err + } + + return nil +} + +type MockFlowAccessAPI struct { + access.AccessAPIServer +} + +// Ping is used to check if the access node is alive and healthy. +func (p MockFlowAccessAPI) Ping(context.Context, *access.PingRequest) (*access.PingResponse, error) { + return &access.PingResponse{}, nil +} diff --git a/bors.toml b/bors.toml index 5b8ba4e5df2..acb31db6ed9 100644 --- a/bors.toml +++ b/bors.toml @@ -1,4 +1,4 @@ -# See https://forum.bors.tech/t/bug-wildcard-status-ignores-1-match/438 +# See https://forum.bors.tech/t/bug-wildcard-status-ignores-1-match/438 # for why we need to explicitly list all statuses status = [ diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index f1b93029f5d..ee8eb943032 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/go-bitswap" @@ -29,6 +30,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/committees" consensuspubsub "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/signature" + hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" @@ -152,6 +154,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { PreferredExecutionNodeIDs: nil, FixedExecutionNodeIDs: nil, MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, ExecutionNodeAddress: "localhost:9000", logTxTimeToFinalized: false, @@ -202,10 +205,11 @@ type FlowAccessNodeBuilder struct { TransactionMetrics module.TransactionMetrics AccessMetrics module.AccessMetrics PingMetrics module.PingMetrics - Committee hotstuff.Committee + Committee hotstuff.DynamicCommittee Finalized *flow.Header Pending []*flow.Header FollowerCore module.HotStuffFollower + Validator hotstuff.Validator ExecutionDataDownloader execution_data.Downloader ExecutionDataRequester state_synchronization.ExecutionDataRequester ExecutionDataStore execution_data.ExecutionDataStore @@ -250,7 +254,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilde func (builder *FlowAccessNodeBuilder) buildSyncCore() *FlowAccessNodeBuilder { builder.Module("sync core", func(node *cmd.NodeConfig) error { - syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) builder.SyncCore = syncCore return err @@ -260,14 +264,15 @@ func (builder *FlowAccessNodeBuilder) buildSyncCore() *FlowAccessNodeBuilder { } func (builder *FlowAccessNodeBuilder) buildCommittee() *FlowAccessNodeBuilder { - builder.Module("committee", func(node *cmd.NodeConfig) error { + builder.Component("committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state - // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee + // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS committee // Note: node.Me.NodeID() is not part of the consensus committee committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(committee) builder.Committee = committee - return err + return committee, err }) return builder @@ -293,6 +298,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerCore() *FlowAccessNodeBuilder packer := signature.NewConsensusSigDataPacker(builder.Committee) // initialize the verifier for the protocol consensus verifier := verification.NewCombinedVerifier(builder.Committee, packer) + builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) followerCore, err := consensus.NewFollower( node.Logger, @@ -335,6 +341,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild builder.FollowerState, conCache, builder.FollowerCore, + builder.Validator, builder.SyncCore, node.Tracer, follower.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold)), @@ -592,8 +599,9 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.DurationVar(&builder.rpcConf.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.CollectionClientTimeout, "grpc client timeout for a collection node") flags.DurationVar(&builder.rpcConf.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.ExecutionClientTimeout, "grpc client timeout for an execution node") flags.UintVar(&builder.rpcConf.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") - flags.IntVar(&builder.rpcConf.MaxExecutionDataMsgSize, "max-block-msg-size", defaultConfig.rpcConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data") + flags.UintVar(&builder.rpcConf.MaxExecutionDataMsgSize, "max-block-msg-size", defaultConfig.rpcConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data") flags.StringSliceVar(&builder.rpcConf.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") flags.StringSliceVar(&builder.rpcConf.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized") @@ -697,7 +705,8 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() { // The following wrapper allows to black-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of blocked nodes to true - blocklistWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB) + builder.NodeBlockListDistributor = cache.NewNodeBlockListDistributor() + blocklistWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeBlockListDistributor) if err != nil { return fmt.Errorf("could not initialize NodeBlocklistWrapper: %w", err) } @@ -752,6 +761,8 @@ func (builder *FlowAccessNodeBuilder) Initialize() error { builder.EnqueueTracer() builder.PreInit(cmd.DynamicStartPreInit) + builder.ValidateRootSnapshot(badgerState.ValidRootSnapshotContainsEntityExpiryRange) + return nil } @@ -786,8 +797,8 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { collectionRPCConn, err := grpc.Dial( builder.rpcConf.CollectionAddr, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithInsecure(), //nolint:staticcheck + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials()), backend.WithClientUnaryInterceptor(builder.rpcConf.CollectionClientTimeout)) if err != nil { return err @@ -805,8 +816,8 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { historicalAccessRPCConn, err := grpc.Dial( addr, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithInsecure()) //nolint:staticcheck + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return err } @@ -1056,7 +1067,10 @@ func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() { // - Default Flow libp2p pubsub options func (builder *FlowAccessNodeBuilder) initLibP2PFactory(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) p2pbuilder.LibP2PFactoryFunc { return func() (p2p.LibP2PNode, error) { - connManager := connection.NewConnManager(builder.Logger, networkMetrics) + connManager, err := connection.NewConnManager(builder.Logger, networkMetrics, builder.ConnectionManagerConfig) + if err != nil { + return nil, fmt.Errorf("could not create connection manager: %w", err) + } libp2pNode, err := p2pbuilder.NewNodeBuilder( builder.Logger, @@ -1099,13 +1113,11 @@ func (builder *FlowAccessNodeBuilder) initLibP2PFactory(networkKey crypto.Privat func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, networkMetrics module.NetworkSecurityMetrics, libp2pNode p2p.LibP2PNode, - validators ...network.MessageValidator) network.Middleware { - + validators ...network.MessageValidator, +) network.Middleware { logger := builder.Logger.With().Bool("staked", false).Logger() - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(logger, networkMetrics) - - builder.Middleware = middleware.NewMiddleware( + mw := middleware.NewMiddleware( logger, libp2pNode, nodeID, @@ -1115,7 +1127,9 @@ func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, builder.IDTranslator, builder.CodecFactory(), slashingViolationsConsumer, - middleware.WithMessageValidators(validators...)) - + middleware.WithMessageValidators(validators...), // use default identifier provider + ) + builder.NodeBlockListDistributor.AddConsumer(mw) + builder.Middleware = mw return builder.Middleware } diff --git a/cmd/bootstrap/cmd/access_keygen.go b/cmd/bootstrap/cmd/access_keygen.go new file mode 100644 index 00000000000..ecd0c5d3945 --- /dev/null +++ b/cmd/bootstrap/cmd/access_keygen.go @@ -0,0 +1,136 @@ +package cmd + +import ( + "crypto/ecdsa" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/bootstrap" + "github.com/onflow/flow-go/utils/grpcutils" +) + +const certValidityPeriod = 100 * 365 * 24 * time.Hour // ~100 years + +var ( + flagSANs string + flagCommonName string + flagNodeInfoFile string + flagOutputKeyFile string + flagOutputCertFile string +) + +var accessKeyCmd = &cobra.Command{ + Use: "access-keygen", + Short: "Generate access node grpc TLS key and certificate", + Run: accessKeyCmdRun, +} + +func init() { + rootCmd.AddCommand(accessKeyCmd) + + accessKeyCmd.Flags().StringVar(&flagNodeInfoFile, "node-info", "", "path to node's node-info.priv.json file") + _ = accessKeyCmd.MarkFlagRequired("node-info") + + accessKeyCmd.Flags().StringVar(&flagOutputKeyFile, "key", "./access-tls.key", "path to output private key file") + accessKeyCmd.Flags().StringVar(&flagOutputCertFile, "cert", "./access-tls.crt", "path to output certificate file") + accessKeyCmd.Flags().StringVar(&flagCommonName, "cn", "", "common name to include in the certificate") + accessKeyCmd.Flags().StringVar(&flagSANs, "sans", "", "subject alternative names to include in the certificate, comma separated") +} + +// accessKeyCmdRun generate an Access node TLS key and certificate +func accessKeyCmdRun(_ *cobra.Command, _ []string) { + networkKey, err := loadNetworkKey(flagNodeInfoFile) + if err != nil { + log.Fatal().Msgf("could not load node-info file: %v", err) + } + + certTmpl, err := defaultCertTemplate() + if err != nil { + log.Fatal().Msgf("could not create certificate template: %v", err) + } + + if flagCommonName != "" { + log.Info().Msgf("using cn: %s", flagCommonName) + certTmpl.Subject.CommonName = flagCommonName + } + + if flagSANs != "" { + log.Info().Msgf("using SANs: %s", flagSANs) + certTmpl.DNSNames = strings.Split(flagSANs, ",") + } + + cert, err := grpcutils.X509Certificate(networkKey, grpcutils.WithCertTemplate(certTmpl)) + if err != nil { + log.Fatal().Msgf("could not generate key pair: %v", err) + } + + // write cert and private key to disk + keyBytes, err := x509.MarshalECPrivateKey(cert.PrivateKey.(*ecdsa.PrivateKey)) + if err != nil { + log.Fatal().Msgf("could not encode private key: %v", err) + } + + err = os.WriteFile(flagOutputKeyFile, pem.EncodeToMemory(&pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: keyBytes, + }), 0600) + if err != nil { + log.Fatal().Msgf("could not write private key: %v", err) + } + + err = os.WriteFile(flagOutputCertFile, pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Certificate[0], + }), 0600) + if err != nil { + log.Fatal().Msgf("could not write certificate: %v", err) + } +} + +func loadNetworkKey(nodeInfoPath string) (crypto.PrivateKey, error) { + data, err := os.ReadFile(nodeInfoPath) + if err != nil { + return nil, fmt.Errorf("could not read private node info (path=%s): %w", nodeInfoPath, err) + } + + var info bootstrap.NodeInfoPriv + err = json.Unmarshal(data, &info) + if err != nil { + return nil, fmt.Errorf("could not parse private node info (path=%s): %w", nodeInfoPath, err) + } + + return info.NetworkPrivKey.PrivateKey, nil +} + +func defaultCertTemplate() (*x509.Certificate, error) { + bigNum := big.NewInt(1 << 62) + sn, err := rand.Int(rand.Reader, bigNum) + if err != nil { + return nil, err + } + + subjectSN, err := rand.Int(rand.Reader, bigNum) + if err != nil { + return nil, err + } + + return &x509.Certificate{ + SerialNumber: sn, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(certValidityPeriod), + // According to RFC 3280, the issuer field must be set, + // see https://datatracker.ietf.org/doc/html/rfc3280#section-4.1.2.4. + Subject: pkix.Name{SerialNumber: subjectSN.String()}, + }, nil +} diff --git a/cmd/bootstrap/cmd/access_keygen_test.go b/cmd/bootstrap/cmd/access_keygen_test.go new file mode 100644 index 00000000000..67c6ea55f66 --- /dev/null +++ b/cmd/bootstrap/cmd/access_keygen_test.go @@ -0,0 +1,95 @@ +package cmd + +import ( + "crypto/ecdsa" + "crypto/x509" + "encoding/asn1" + "encoding/pem" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/utils/unittest" +) + +func TestAccessKeyFileCreated(t *testing.T) { + unittest.RunWithTempDir(t, func(bootDir string) { + hook := zeroLoggerHook{logs: &strings.Builder{}} + log = log.Hook(hook) + + // generate test node keys + flagRole = "access" + flagAddress = "localhost:1234" + flagOutdir = bootDir + + keyCmdRun(nil, nil) + + // find the node-info.priv.json file + // the path includes a random hex string, so we need to find it + err := filepath.Walk(bootDir, func(path string, info os.FileInfo, err error) error { + if err == nil && info.Name() == "node-info.priv.json" { + flagNodeInfoFile = path + } + return nil + }) + require.NoError(t, err) + + sans := []string{"unittest1.onflow.org", "unittest2.onflow.org"} + + flagSANs = strings.Join(sans, ",") + flagCommonName = "unittest.onflow.org" + flagOutputKeyFile = filepath.Join(bootDir, "test-access-key.key") + flagOutputCertFile = filepath.Join(bootDir, "test-access-key.cert") + + // run command with flags + accessKeyCmdRun(nil, nil) + + // make sure key/cert files exists (regex checks this too) + require.FileExists(t, flagOutputKeyFile) + require.FileExists(t, flagOutputCertFile) + + // decode key and cert and make sure they match + keyData, err := os.ReadFile(flagOutputKeyFile) + require.NoError(t, err) + + certData, err := os.ReadFile(flagOutputCertFile) + require.NoError(t, err) + + privKey, cert := decodeKeys(t, keyData, certData) + + // check that the public key from the cert matches the private key + ecdsaPubKey, ok := cert.PublicKey.(*ecdsa.PublicKey) + require.True(t, ok) + require.Equal(t, privKey.PublicKey, *ecdsaPubKey) + + // check that the common name and subject alt names are correct + assert.Equal(t, flagCommonName, cert.Subject.CommonName, "expected %s, got %s", flagCommonName, cert.Subject.CommonName) + assert.Equal(t, flagCommonName, cert.Issuer.CommonName, "expected %s, got %s", flagCommonName, cert.Issuer.CommonName) + assert.ElementsMatch(t, sans, cert.DNSNames) + + // check that the libp2p extension is present + found := false + for _, ext := range cert.Extensions { + if ext.Id.Equal(asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 53594, 1, 1}) { + found = true + } + } + assert.True(t, found, "expected to find libp2p extension") + }) +} + +func decodeKeys(t *testing.T, pemEncoded []byte, pemEncodedPub []byte) (*ecdsa.PrivateKey, *x509.Certificate) { + block, _ := pem.Decode(pemEncoded) + privateKey, err := x509.ParseECPrivateKey(block.Bytes) + require.NoError(t, err) + + blockPub, _ := pem.Decode(pemEncodedPub) + cert, err := x509.ParseCertificate(blockPub.Bytes) + require.NoError(t, err) + + return privateKey, cert +} diff --git a/cmd/bootstrap/cmd/check_machine_account.go b/cmd/bootstrap/cmd/check_machine_account.go index c7cc7b7c0a2..e2261012219 100644 --- a/cmd/bootstrap/cmd/check_machine_account.go +++ b/cmd/bootstrap/cmd/check_machine_account.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" sdk "github.com/onflow/flow-go-sdk" client "github.com/onflow/flow-go-sdk/access/grpc" @@ -62,7 +63,7 @@ func checkMachineAccountRun(_ *cobra.Command, _ []string) { Str("hash_algo", machineAccountInfo.HashAlgorithm.String()). Msg("read machine account info from disk") - flowClient, err := client.NewClient(flagAccessAPIAddress, grpc.WithInsecure()) //nolint:staticcheck + flowClient, err := client.NewClient(flagAccessAPIAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatal().Err(err).Msgf("could not connect to access API at address %s", flagAccessAPIAddress) } diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 7ca264935b5..5d1eb74106a 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/module/epochs" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/inmem" "github.com/onflow/flow-go/utils/io" @@ -46,6 +47,7 @@ var ( flagNumViewsInEpoch uint64 flagNumViewsInStakingAuction uint64 flagNumViewsInDKGPhase uint64 + flagEpochCommitSafetyThreshold uint64 // this flag is used to seed the DKG, clustering and cluster QC generation flagBootstrapRandomSeed []byte @@ -98,6 +100,7 @@ func addFinalizeCmdFlags() { finalizeCmd.Flags().Uint64Var(&flagNumViewsInEpoch, "epoch-length", 4000, "length of each epoch measured in views") finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views") finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views") + finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline") finalizeCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation") finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork") @@ -108,6 +111,7 @@ func addFinalizeCmdFlags() { cmd.MarkFlagRequired(finalizeCmd, "epoch-length") cmd.MarkFlagRequired(finalizeCmd, "epoch-staking-phase-length") cmd.MarkFlagRequired(finalizeCmd, "epoch-dkg-phase-length") + cmd.MarkFlagRequired(finalizeCmd, "epoch-commit-safety-threshold") cmd.MarkFlagRequired(finalizeCmd, "protocol-version") // optional parameters to influence various aspects of identity generation @@ -133,6 +137,12 @@ func finalize(cmd *cobra.Command, args []string) { } } + // validate epoch configs + err := validateEpochConfig() + if err != nil { + log.Fatal().Err(err).Msg("invalid or unsafe epoch commit threshold config") + } + if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength { log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid") return @@ -215,7 +225,7 @@ func finalize(cmd *cobra.Command, args []string) { // construct serializable root protocol snapshot log.Info().Msg("constructing root protocol snapshot") - snapshot, err := inmem.SnapshotFromBootstrapStateWithProtocolVersion(block, result, seal, rootQC, flagProtocolVersion) + snapshot, err := inmem.SnapshotFromBootstrapStateWithParams(block, result, seal, rootQC, flagProtocolVersion, flagEpochCommitSafetyThreshold) if err != nil { log.Fatal().Err(err).Msg("unable to generate root protocol snapshot") } @@ -634,3 +644,30 @@ func generateEmptyExecutionState( log.Info().Msg("") return } + +// validateEpochConfig validates configuration of the epoch commitment deadline. +func validateEpochConfig() error { + chainID := parseChainID(flagRootChain) + dkgFinalView := flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 // 3 DKG phases + epochCommitDeadline := flagNumViewsInEpoch - flagEpochCommitSafetyThreshold + + defaultSafetyThreshold, err := protocol.DefaultEpochCommitSafetyThreshold(chainID) + if err != nil { + return fmt.Errorf("could not get default epoch commit safety threshold: %w", err) + } + + // sanity check: the safety threshold is >= the default for the chain + if flagEpochCommitSafetyThreshold < defaultSafetyThreshold { + return fmt.Errorf("potentially unsafe epoch config: epoch commit safety threshold smaller than expected (%d < %d)", flagEpochCommitSafetyThreshold, defaultSafetyThreshold) + } + // sanity check: epoch commitment deadline cannot be before the DKG end + if epochCommitDeadline <= dkgFinalView { + return fmt.Errorf("invalid epoch config: the epoch commitment deadline (%d) is before the DKG final view (%d)", epochCommitDeadline, dkgFinalView) + } + // sanity check: the difference between DKG end and safety threshold is also >= the default safety threshold + if epochCommitDeadline-dkgFinalView < defaultSafetyThreshold { + return fmt.Errorf("potentially unsafe epoch config: time between DKG end and epoch commitment deadline is smaller than expected (%d-%d < %d)", + epochCommitDeadline, dkgFinalView, defaultSafetyThreshold) + } + return nil +} diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 6b25a8b2749..033e29b6609 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -81,6 +81,10 @@ func TestFinalize_HappyPath(t *testing.T) { flagRootCommit = hex.EncodeToString(rootCommit[:]) flagEpochCounter = epochCounter + flagNumViewsInEpoch = 100_000 + flagNumViewsInStakingAuction = 50_000 + flagNumViewsInDKGPhase = 2_000 + flagEpochCommitSafetyThreshold = 1_000 flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) @@ -122,6 +126,10 @@ func TestFinalize_Deterministic(t *testing.T) { flagRootChain = chainName flagRootHeight = rootHeight flagEpochCounter = epochCounter + flagNumViewsInEpoch = 100_000 + flagNumViewsInStakingAuction = 50_000 + flagNumViewsInDKGPhase = 2_000 + flagEpochCommitSafetyThreshold = 1_000 // set deterministic bootstrapping seed flagBootstrapRandomSeed = deterministicSeed @@ -197,6 +205,10 @@ func TestFinalize_SameSeedDifferentStateCommits(t *testing.T) { flagRootChain = chainName flagRootHeight = rootHeight flagEpochCounter = epochCounter + flagNumViewsInEpoch = 100_000 + flagNumViewsInStakingAuction = 50_000 + flagNumViewsInDKGPhase = 2_000 + flagEpochCommitSafetyThreshold = 1_000 // set deterministic bootstrapping seed flagBootstrapRandomSeed = deterministicSeed @@ -303,6 +315,10 @@ func TestFinalize_InvalidRandomSeedLength(t *testing.T) { flagRootChain = chainName flagRootHeight = rootHeight flagEpochCounter = epochCounter + flagNumViewsInEpoch = 100_000 + flagNumViewsInStakingAuction = 50_000 + flagNumViewsInDKGPhase = 2_000 + flagEpochCommitSafetyThreshold = 1_000 // set deterministic bootstrapping seed flagBootstrapRandomSeed = deterministicSeed diff --git a/cmd/bootstrap/cmd/qc.go b/cmd/bootstrap/cmd/qc.go index e38a07b271f..6e97363051b 100644 --- a/cmd/bootstrap/cmd/qc.go +++ b/cmd/bootstrap/cmd/qc.go @@ -20,7 +20,13 @@ func constructRootQC(block *flow.Block, votes []*model.Vote, allNodes, internalN log.Fatal().Err(err).Msg("failed to generate QC participant data") } - qc, err := run.GenerateRootQC(block, votes, participantData, identities) + qc, invalidVotesErr, err := run.GenerateRootQC(block, votes, participantData, identities) + if len(invalidVotesErr) > 0 { + for _, err := range invalidVotesErr { + log.Warn().Err(err).Msg("invalid vote") + } + } + if err != nil { log.Fatal().Err(err).Msg("generating root QC failed") } diff --git a/cmd/bootstrap/run/cluster_qc.go b/cmd/bootstrap/run/cluster_qc.go index c8db89ab92e..fa91e5cc4f8 100644 --- a/cmd/bootstrap/run/cluster_qc.go +++ b/cmd/bootstrap/run/cluster_qc.go @@ -7,7 +7,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" @@ -59,17 +58,16 @@ func GenerateClusterRootQC(signers []bootstrap.NodeInfo, allCommitteeMembers flo if err != nil { return nil, fmt.Errorf("could not create cluster validator: %w", err) } - err = val.ValidateQC(createdQC, clusterRootBlock) + err = val.ValidateQC(createdQC) return createdQC, err } // createClusterValidator creates validator for cluster consensus -func createClusterValidator(committee hotstuff.Committee) (hotstuff.Validator, error) { +func createClusterValidator(committee hotstuff.DynamicCommittee) (hotstuff.Validator, error) { verifier := verification.NewStakingVerifier() - forks := &mocks.ForksReader{} - hotstuffValidator := validator.New(committee, forks, verifier) + hotstuffValidator := validator.New(committee, verifier) return hotstuffValidator, nil } diff --git a/cmd/bootstrap/run/qc.go b/cmd/bootstrap/run/qc.go index 8c2617561f7..c07879eb446 100644 --- a/cmd/bootstrap/run/qc.go +++ b/cmd/bootstrap/run/qc.go @@ -7,7 +7,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" hotstuffSig "github.com/onflow/flow-go/consensus/hotstuff/signature" "github.com/onflow/flow-go/consensus/hotstuff/validator" @@ -45,11 +44,18 @@ func (pd *ParticipantData) Identities() flow.IdentityList { // which assumes that only consensus participants construct QCs, which also have produce votes. // // TODO: modularize QC construction code (and code to verify QC) to be instantiated without needing private keys. -func GenerateRootQC(block *flow.Block, votes []*model.Vote, participantData *ParticipantData, identities flow.IdentityList) (*flow.QuorumCertificate, error) { +// It returns (qc, nil, nil) if a QC can be constructed with enough votes, and there is no invalid votes +// It returns (qc, invalidVotes, nil) if there are some invalid votes, but a QC can still be constructed +// It returns (nil, invalidVotes, err) if no qc can be constructed with not enough votes or running any any exception +func GenerateRootQC(block *flow.Block, votes []*model.Vote, participantData *ParticipantData, identities flow.IdentityList) ( + *flow.QuorumCertificate, // the constructed QC + []error, // return invalid votes error + error, // exception or could not construct qc +) { // create consensus committee's state committee, err := committees.NewStaticCommittee(identities, flow.Identifier{}, participantData.Lookup, participantData.GroupKey) if err != nil { - return nil, err + return nil, nil, err } // STEP 1: create VoteProcessor @@ -59,34 +65,44 @@ func GenerateRootQC(block *flow.Block, votes []*model.Vote, participantData *Par createdQC = qc }) if err != nil { - return nil, fmt.Errorf("could not CombinedVoteProcessor processor: %w", err) + return nil, nil, fmt.Errorf("could not CombinedVoteProcessor processor: %w", err) } + invalidVotes := make([]error, 0, len(votes)) // STEP 2: feed the votes into the vote processor to create QC for _, vote := range votes { err := processor.Process(vote) + + // in case there are invalid votes, we continue process more votes, + // so that finalizing block won't be interrupted by any invalid vote. + // if no enough votes are collected, finalize will fail and exit anyway, because + // no QC will be built. if err != nil { - return nil, fmt.Errorf("fail to process vote %v for block %v from signer %v: %w", + if model.IsInvalidVoteError(err) { + invalidVotes = append(invalidVotes, err) + continue + } + return nil, invalidVotes, fmt.Errorf("fail to process vote %v for block %v from signer %v: %w", vote.ID(), - block.ID(), + vote.BlockID, vote.SignerID, err) } } if createdQC == nil { - return nil, fmt.Errorf("QC is not created, total number of votes %v, expect to have 2/3 votes of %v participants", + return nil, invalidVotes, fmt.Errorf("QC is not created, total number of votes %v, expect to have 2/3 votes of %v participants", len(votes), len(identities)) } // STEP 3: validate constructed QC val, err := createValidator(committee) if err != nil { - return nil, err + return nil, invalidVotes, err } - err = val.ValidateQC(createdQC, hotBlock) + err = val.ValidateQC(createdQC) - return createdQC, err + return createdQC, invalidVotes, err } // GenerateRootBlockVotes generates votes for root block based on participantData @@ -121,12 +137,11 @@ func GenerateRootBlockVotes(block *flow.Block, participantData *ParticipantData) } // createValidator creates validator that can validate votes and QC -func createValidator(committee hotstuff.Committee) (hotstuff.Validator, error) { +func createValidator(committee hotstuff.DynamicCommittee) (hotstuff.Validator, error) { packer := hotstuffSig.NewConsensusSigDataPacker(committee) verifier := verification.NewCombinedVerifier(committee, packer) - forks := &mocks.ForksReader{} - hotstuffValidator := validator.New(committee, forks, verifier) + hotstuffValidator := validator.New(committee, verifier) return hotstuffValidator, nil } diff --git a/cmd/bootstrap/run/qc_test.go b/cmd/bootstrap/run/qc_test.go index afc8849d329..5deed36d1ed 100644 --- a/cmd/bootstrap/run/qc_test.go +++ b/cmd/bootstrap/run/qc_test.go @@ -17,19 +17,31 @@ import ( func TestGenerateRootQC(t *testing.T) { participantData := createSignerData(t, 3) - block := unittest.BlockFixture() - block.Payload.Guarantees = nil - block.Payload.Seals = nil - block.Header.Height = 0 - block.Header.ParentID = flow.ZeroID - block.Header.View = 3 - block.Header.PayloadHash = block.Payload.Hash() - - votes, err := GenerateRootBlockVotes(&block, participantData) + block := unittest.GenesisFixture() + + votes, err := GenerateRootBlockVotes(block, participantData) + require.NoError(t, err) + + _, invalid, err := GenerateRootQC(block, votes, participantData, participantData.Identities()) require.NoError(t, err) + require.Len(t, invalid, 0) // no invalid votes +} + +func TestGenerateRootQCWithSomeInvalidVotes(t *testing.T) { + participantData := createSignerData(t, 10) + + block := unittest.GenesisFixture() + + votes, err := GenerateRootBlockVotes(block, participantData) + require.NoError(t, err) + + // make 2 votes invalid + votes[0].SigData = unittest.SignatureFixture() // make invalid signature + votes[1].SignerID = unittest.IdentifierFixture() // make invalid signer - _, err = GenerateRootQC(&block, votes, participantData, participantData.Identities()) + _, invalid, err := GenerateRootQC(block, votes, participantData, participantData.Identities()) require.NoError(t, err) + require.Len(t, invalid, 2) // 2 invalid votes } func createSignerData(t *testing.T, n int) *ParticipantData { diff --git a/cmd/bootstrap/transit/cmd/snapshot.go b/cmd/bootstrap/transit/cmd/snapshot.go index 939fe380a24..b918779bb7f 100644 --- a/cmd/bootstrap/transit/cmd/snapshot.go +++ b/cmd/bootstrap/transit/cmd/snapshot.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" client "github.com/onflow/flow-go-sdk/access/grpc" "github.com/onflow/flow-go/engine/common/rpc/convert" @@ -52,7 +53,7 @@ func snapshot(cmd *cobra.Command, args []string) { } // create a flow client with given access address - flowClient, err := client.NewClient(flagAccessAddress, grpc.WithInsecure()) //nolint:staticcheck + flowClient, err := client.NewClient(flagAccessAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatal().Err(err).Msg("could not create flow client") } diff --git a/cmd/collection/main.go b/cmd/collection/main.go index 906eb94945d..4ca8a48e6b6 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -7,12 +7,18 @@ import ( "github.com/spf13/pflag" client "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go/cmd/util/cmd/common" + "github.com/onflow/flow-go/consensus/hotstuff/validator" + "github.com/onflow/flow-go/model/bootstrap" + modulecompliance "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/mempool/queue" + "github.com/onflow/flow-go/utils/grpcutils" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" "github.com/onflow/flow-go/cmd" - "github.com/onflow/flow-go/cmd/util/cmd/common" "github.com/onflow/flow-go/consensus" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" @@ -28,19 +34,16 @@ import ( "github.com/onflow/flow-go/engine/common/provider" consync "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/fvm/systemcontracts" - "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/buffer" builder "github.com/onflow/flow-go/module/builder/collection" "github.com/onflow/flow-go/module/chainsync" - modulecompliance "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/epochs" confinalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/mempool" epochpool "github.com/onflow/flow-go/module/mempool/epochs" - "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" @@ -53,25 +56,24 @@ import ( func main() { var ( - txLimit uint - maxCollectionSize uint - maxCollectionByteSize uint64 - maxCollectionTotalGas uint64 - maxCollectionRequestCacheSize uint32 // collection provider engine - collectionProviderWorkers uint // collection provider engine - builderExpiryBuffer uint - builderPayerRateLimitDryRun bool - builderPayerRateLimit float64 - builderUnlimitedPayers []string - hotstuffTimeout time.Duration - hotstuffMinTimeout time.Duration - hotstuffTimeoutIncreaseFactor float64 - hotstuffTimeoutDecreaseFactor float64 - hotstuffTimeoutVoteAggregationFraction float64 - blockRateDelay time.Duration - startupTimeString string - startupTime time.Time - + txLimit uint + maxCollectionSize uint + maxCollectionByteSize uint64 + maxCollectionTotalGas uint64 + maxCollectionRequestCacheSize uint32 // collection provider engine + collectionProviderWorkers uint // collection provider engine + builderExpiryBuffer uint + builderPayerRateLimitDryRun bool + builderPayerRateLimit float64 + builderUnlimitedPayers []string + hotstuffMinTimeout time.Duration + hotstuffTimeoutAdjustmentFactor float64 + hotstuffHappyPathMaxRoundFailures uint64 + blockRateDelay time.Duration + startupTimeString string + startupTime time.Time + + mainConsensusCommittee *committees.Consensus followerState protocol.MutableState ingestConf = ingest.DefaultConfig() rpcConf rpc.Config @@ -85,6 +87,7 @@ func main() { push *pusher.Engine ing *ingest.Engine mainChainSyncCore *chainsync.Core + followerCore *hotstuff.FollowerLoop // follower hotstuff logic followerEng *followereng.Engine colMetrics module.CollectionMetrics err error @@ -104,6 +107,8 @@ func main() { "maximum number of transactions in the memory pool") flags.StringVarP(&rpcConf.ListenAddr, "ingress-addr", "i", "localhost:9000", "the address the ingress server listens on") + flags.UintVar(&rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, + "the maximum message size in bytes for messages sent or received over grpc") flags.BoolVar(&rpcConf.RpcMetricsEnabled, "rpc-metrics-enabled", false, "whether to enable the rpc metrics") flags.Uint64Var(&ingestConf.MaxGasLimit, "ingest-max-gas-limit", flow.DefaultMaxTransactionGasLimit, @@ -132,19 +137,12 @@ func main() { "maximum byte size of the proposed collection") flags.Uint64Var(&maxCollectionTotalGas, "builder-max-collection-total-gas", flow.DefaultMaxCollectionTotalGas, "maximum total amount of maxgas of transactions in proposed collections") - flags.DurationVar(&hotstuffTimeout, "hotstuff-timeout", 60*time.Second, - "the initial timeout for the hotstuff pacemaker") flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 2500*time.Millisecond, - "the lower timeout bound for the hotstuff pacemaker") - flags.Float64Var(&hotstuffTimeoutIncreaseFactor, "hotstuff-timeout-increase-factor", - timeout.DefaultConfig.TimeoutIncrease, - "multiplicative increase of timeout value in case of time out event") - flags.Float64Var(&hotstuffTimeoutDecreaseFactor, "hotstuff-timeout-decrease-factor", - timeout.DefaultConfig.TimeoutDecrease, - "multiplicative decrease of timeout value in case of progress") - flags.Float64Var(&hotstuffTimeoutVoteAggregationFraction, "hotstuff-timeout-vote-aggregation-fraction", - timeout.DefaultConfig.VoteAggregationTimeoutFraction, - "additional fraction of replica timeout that the primary will wait for votes") + "the lower timeout bound for the hotstuff pacemaker, this is also used as initial timeout") + flags.Float64Var(&hotstuffTimeoutAdjustmentFactor, "hotstuff-timeout-adjustment-factor", timeout.DefaultConfig.TimeoutAdjustmentFactor, + "adjustment of timeout duration in case of time out event") + flags.Uint64Var(&hotstuffHappyPathMaxRoundFailures, "hotstuff-happy-path-max-round-failures", timeout.DefaultConfig.HappyPathMaxRoundFailures, + "number of failed rounds before first timeout increase") flags.DurationVar(&blockRateDelay, "block-rate-delay", 250*time.Millisecond, "the delay to broadcast block proposal in order to control block production rate") flags.Uint64Var(&clusterComplianceConfig.SkipNewProposalsThreshold, @@ -217,7 +215,8 @@ func main() { return nil }). Module("main chain sync core", func(node *cmd.NodeConfig) error { - mainChainSyncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + log := node.Logger.With().Str("sync_chain_id", node.RootChainID.String()).Logger() + mainChainSyncCore, err = chainsync.New(log, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err }). Module("machine account config", func(node *cmd.NodeConfig) error { @@ -259,36 +258,28 @@ func main() { return validator, err }). - Component("follower engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - - // initialize cleaner for DB - cleaner := storagekv.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) - - // create a finalizer that will handling updating the protocol - // state when the follower detects newly finalized blocks - finalizer := confinalizer.NewFinalizer(node.DB, node.Storage.Headers, followerState, node.Tracer) - + Component("consensus committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee // Note: node.Me.NodeID() is not part of the consensus committee - mainConsensusCommittee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + mainConsensusCommittee, err = committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(mainConsensusCommittee) + return mainConsensusCommittee, err + }). + Component("follower core", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + // create a finalizer for updating the protocol + // state when the follower detects newly finalized blocks + finalizer := confinalizer.NewFinalizer(node.DB, node.Storage.Headers, followerState, node.Tracer) + finalized, pending, err := recovery.FindLatest(node.State, node.Storage.Headers) if err != nil { - return nil, fmt.Errorf("could not create Committee state for main consensus: %w", err) + return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) } - packer := hotsignature.NewConsensusSigDataPacker(mainConsensusCommittee) // initialize the verifier for the protocol consensus verifier := verification.NewCombinedVerifier(mainConsensusCommittee, packer) - finalizationDistributor = pubsub.NewFinalizationDistributor() - - finalized, pending, err := recovery.FindLatest(node.State, node.Storage.Headers) - if err != nil { - return nil, fmt.Errorf("could not find latest finalized block and pending blocks to recover consensus follower: %w", err) - } - // creates a consensus follower with noop consumer as the notifier - followerCore, err := consensus.NewFollower( + followerCore, err = consensus.NewFollower( node.Logger, mainConsensusCommittee, node.Storage.Headers, @@ -303,6 +294,17 @@ func main() { if err != nil { return nil, fmt.Errorf("could not create follower core logic: %w", err) } + return followerCore, nil + }). + Component("follower engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + // initialize cleaner for DB + cleaner := storagekv.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) + + packer := hotsignature.NewConsensusSigDataPacker(mainConsensusCommittee) + // initialize the verifier for the protocol consensus + verifier := verification.NewCombinedVerifier(mainConsensusCommittee, packer) + + validator := validator.New(mainConsensusCommittee, verifier) followerEng, err = followereng.New( node.Logger, @@ -316,6 +318,7 @@ func main() { followerState, followerBuffer, followerCore, + validator, mainChainSyncCore, node.Tracer, followereng.WithComplianceOptions(modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), @@ -452,7 +455,7 @@ func main() { return nil, err } - proposalFactory, err := factories.NewProposalEngineFactory( + complianceEngineFactory, err := factories.NewComplianceEngineFactory( node.Logger, node.Network, node.Me, @@ -467,12 +470,16 @@ func main() { return nil, err } + syncCoreFactory, err := factories.NewSyncCoreFactory(node.Logger, node.SyncCoreConfig) + if err != nil { + return nil, err + } + syncFactory, err := factories.NewSyncEngineFactory( node.Logger, node.Metrics.Engine, node.Network, node.Me, - node.SyncCoreConfig, ) if err != nil { return nil, err @@ -484,11 +491,9 @@ func main() { opts := []consensus.Option{ consensus.WithBlockRateDelay(blockRateDelay), - consensus.WithInitialTimeout(hotstuffTimeout), consensus.WithMinTimeout(hotstuffMinTimeout), - consensus.WithVoteAggregationTimeoutFraction(hotstuffTimeoutVoteAggregationFraction), - consensus.WithTimeoutIncreaseFactor(hotstuffTimeoutIncreaseFactor), - consensus.WithTimeoutDecreaseFactor(hotstuffTimeoutDecreaseFactor), + consensus.WithTimeoutAdjustmentFactor(hotstuffTimeoutAdjustmentFactor), + consensus.WithHappyPathMaxRoundFailures(hotstuffHappyPathMaxRoundFailures), } if !startupTime.IsZero() { @@ -500,6 +505,8 @@ func main() { node.Me, node.DB, node.State, + node.Metrics.Engine, + node.Metrics.Mempool, createMetrics, opts..., ) @@ -523,14 +530,24 @@ func main() { qcContractClients, ) + messageHubFactory := factories.NewMessageHubFactory( + node.Logger, + node.Network, + node.Me, + node.Metrics.Engine, + node.State, + ) + factory := factories.NewEpochComponentsFactory( node.Me, pools, builderFactory, clusterStateFactory, hotstuffFactory, - proposalFactory, + complianceEngineFactory, + syncCoreFactory, syncFactory, + messageHubFactory, ) heightEvents := gadgets.NewHeights() diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 5feb2a1090f..560e4850160 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -24,6 +24,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" "github.com/onflow/flow-go/consensus/hotstuff/persister" hotsignature "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutcollector" "github.com/onflow/flow-go/consensus/hotstuff/verification" "github.com/onflow/flow-go/consensus/hotstuff/votecollector" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" @@ -34,7 +35,7 @@ import ( dkgeng "github.com/onflow/flow-go/engine/consensus/dkg" "github.com/onflow/flow-go/engine/consensus/ingestion" "github.com/onflow/flow-go/engine/consensus/matching" - "github.com/onflow/flow-go/engine/consensus/provider" + "github.com/onflow/flow-go/engine/consensus/message_hub" "github.com/onflow/flow-go/engine/consensus/sealing" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/bootstrap" @@ -54,7 +55,9 @@ import ( consensusMempools "github.com/onflow/flow-go/module/mempool/consensus" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" + msig "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/module/updatable_configs" + "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/module/validation" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" @@ -69,28 +72,26 @@ import ( func main() { var ( - guaranteeLimit uint - resultLimit uint - approvalLimit uint - sealLimit uint - pendingReceiptsLimit uint - minInterval time.Duration - maxInterval time.Duration - maxSealPerBlock uint - maxGuaranteePerBlock uint - hotstuffTimeout time.Duration - hotstuffMinTimeout time.Duration - hotstuffTimeoutIncreaseFactor float64 - hotstuffTimeoutDecreaseFactor float64 - hotstuffTimeoutVoteAggregationFraction float64 - blockRateDelay time.Duration - chunkAlpha uint - requiredApprovalsForSealVerification uint - requiredApprovalsForSealConstruction uint - emergencySealing bool - dkgControllerConfig dkgmodule.ControllerConfig - startupTimeString string - startupTime time.Time + guaranteeLimit uint + resultLimit uint + approvalLimit uint + sealLimit uint + pendingReceiptsLimit uint + minInterval time.Duration + maxInterval time.Duration + maxSealPerBlock uint + maxGuaranteePerBlock uint + hotstuffMinTimeout time.Duration + hotstuffTimeoutAdjustmentFactor float64 + hotstuffHappyPathMaxRoundFailures uint64 + blockRateDelay time.Duration + chunkAlpha uint + requiredApprovalsForSealVerification uint + requiredApprovalsForSealConstruction uint + emergencySealing bool + dkgControllerConfig dkgmodule.ControllerConfig + startupTimeString string + startupTime time.Time // DKG contract client machineAccountInfo *bootstrap.NodeMachineAccountInfo @@ -105,10 +106,10 @@ func main() { receipts mempool.ExecutionTree seals mempool.IncorporatedResultSeals pendingReceipts mempool.PendingReceipts - prov *provider.Engine receiptRequester *requester.Engine syncCore *chainsync.Core comp *compliance.Engine + hot module.HotStuff conMetrics module.ConsensusMetrics mainMetrics module.HotstuffMetrics receiptValidator module.ReceiptValidator @@ -117,6 +118,8 @@ func main() { dkgBrokerTunnel *dkgmodule.BrokerTunnel blockTimer protocol.BlockTimer finalizedHeader *synceng.FinalizedHeaderCache + committee *committees.Consensus + epochLookup *epochs.EpochLookup hotstuffModules *consensus.HotstuffModules dkgState *bstorage.DKGState safeBeaconKeys *bstorage.SafeBeaconPrivateKeys @@ -136,11 +139,9 @@ func main() { flags.DurationVar(&maxInterval, "max-interval", 90*time.Second, "the maximum amount of time between two blocks") flags.UintVar(&maxSealPerBlock, "max-seal-per-block", 100, "the maximum number of seals to be included in a block") flags.UintVar(&maxGuaranteePerBlock, "max-guarantee-per-block", 100, "the maximum number of collection guarantees to be included in a block") - flags.DurationVar(&hotstuffTimeout, "hotstuff-timeout", 60*time.Second, "the initial timeout for the hotstuff pacemaker") - flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 2500*time.Millisecond, "the lower timeout bound for the hotstuff pacemaker") - flags.Float64Var(&hotstuffTimeoutIncreaseFactor, "hotstuff-timeout-increase-factor", timeout.DefaultConfig.TimeoutIncrease, "multiplicative increase of timeout value in case of time out event") - flags.Float64Var(&hotstuffTimeoutDecreaseFactor, "hotstuff-timeout-decrease-factor", timeout.DefaultConfig.TimeoutDecrease, "multiplicative decrease of timeout value in case of progress") - flags.Float64Var(&hotstuffTimeoutVoteAggregationFraction, "hotstuff-timeout-vote-aggregation-fraction", 0.6, "additional fraction of replica timeout that the primary will wait for votes") + flags.DurationVar(&hotstuffMinTimeout, "hotstuff-min-timeout", 2500*time.Millisecond, "the lower timeout bound for the hotstuff pacemaker, this is also used as initial timeout") + flags.Float64Var(&hotstuffTimeoutAdjustmentFactor, "hotstuff-timeout-adjustment-factor", timeout.DefaultConfig.TimeoutAdjustmentFactor, "adjustment of timeout duration in case of time out event") + flags.Uint64Var(&hotstuffHappyPathMaxRoundFailures, "hotstuff-happy-path-max-round-failures", timeout.DefaultConfig.HappyPathMaxRoundFailures, "number of failed rounds before first timeout increase") flags.DurationVar(&blockRateDelay, "block-rate-delay", 500*time.Millisecond, "the delay to broadcast block proposal in order to control block production rate") flags.UintVar(&chunkAlpha, "chunk-alpha", flow.DefaultChunkAssignmentAlpha, "number of verifiers that should be assigned to each chunk") flags.UintVar(&requiredApprovalsForSealVerification, "required-verification-seal-approvals", flow.DefaultRequiredApprovalsForSealValidation, "minimum number of approvals that are required to verify a seal") @@ -171,6 +172,7 @@ func main() { nodeBuilder. PreInit(cmd.DynamicStartPreInit). + ValidateRootSnapshot(badgerState.ValidRootSnapshotContainsEntityExpiryRange). Module("consensus node metrics", func(node *cmd.NodeConfig) error { conMetrics = metrics.NewConsensusCollector(node.Tracer, node.MetricsRegisterer) return nil @@ -356,7 +358,7 @@ func main() { return nil }). Module("sync core", func(node *cmd.NodeConfig) error { - syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err }). Module("finalization distributor", func(node *cmd.NodeConfig) error { @@ -486,17 +488,6 @@ func main() { return e, err }). - Component("provider engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - prov, err = provider.New( - node.Logger, - node.Metrics.Engine, - node.Tracer, - node.Network, - node.State, - node.Me, - ) - return prov, err - }). Component("ingestion engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { core := ingestion.NewCore( node.Logger, @@ -517,6 +508,16 @@ func main() { return ing, err }). + Component("hotstuff committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + committee, err = committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(committee) + return committee, err + }). + Component("epoch lookup", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + epochLookup, err = epochs.NewEpochLookup(node.State) + node.ProtocolEvents.AddConsumer(epochLookup) + return epochLookup, err + }). Component("hotstuff modules", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize the block finalizer finalize := finalizer.NewFinalizer( @@ -533,15 +534,9 @@ func main() { )), ) - // initialize Main consensus committee's state - var committee hotstuff.Committee - committee, err = committees.NewConsensusCommittee(node.State, node.Me.NodeID()) - if err != nil { - return nil, fmt.Errorf("could not create Committee state for main consensus: %w", err) - } - committee = committees.NewMetricsWrapper(committee, mainMetrics) // wrapper for measuring time spent determining consensus committee relations + // wrap Main consensus committee with metrics + wrappedCommittee := committees.NewMetricsWrapper(committee, mainMetrics) // wrapper for measuring time spent determining consensus committee relations - epochLookup := epochs.NewEpochLookup(node.State) beaconKeyStore := hotsignature.NewEpochAwareRandomBeaconKeyStore(epochLookup, safeBeaconKeys) // initialize the combined signer for hotstuff @@ -552,12 +547,13 @@ func main() { ) signer = verification.NewMetricsWrapper(signer, mainMetrics) // wrapper for measuring time spent with crypto-related operations + // create consensus logger + logger := createLogger(node.Logger, node.RootChainID) + // initialize a logging notifier for hotstuff notifier := createNotifier( - node.Logger, + logger, mainMetrics, - node.Tracer, - node.RootChainID, ) notifier.AddConsumer(finalizationDistributor) @@ -583,10 +579,14 @@ func main() { } qcDistributor := pubsub.NewQCCreatedDistributor() - validator := consensus.NewValidator(mainMetrics, committee, forks) - voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) + validator := consensus.NewValidator(mainMetrics, wrappedCommittee) + voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(wrappedCommittee, qcDistributor.OnQcConstructedFromVotes) lowestViewForVoteProcessing := finalizedBlock.View + 1 - aggregator, err := consensus.NewVoteAggregator(node.Logger, + voteAggregator, err := consensus.NewVoteAggregator( + logger, + mainMetrics, + node.Metrics.Engine, + node.Metrics.Mempool, lowestViewForVoteProcessing, notifier, voteProcessorFactory, @@ -595,21 +595,45 @@ func main() { return nil, fmt.Errorf("could not initialize vote aggregator: %w", err) } + timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( + logger, + timeoutCollectorDistributor, + committee, + validator, + msig.ConsensusTimeoutTag, + ) + timeoutAggregator, err := consensus.NewTimeoutAggregator( + logger, + mainMetrics, + node.Metrics.Engine, + node.Metrics.Mempool, + notifier, + timeoutProcessorFactory, + timeoutCollectorDistributor, + lowestViewForVoteProcessing, + ) + if err != nil { + return nil, fmt.Errorf("could not initialize timeout aggregator: %w", err) + } + hotstuffModules = &consensus.HotstuffModules{ - Notifier: notifier, - Committee: committee, - Signer: signer, - Persist: persist, - QCCreatedDistributor: qcDistributor, - FinalizationDistributor: finalizationDistributor, - Forks: forks, - Validator: validator, - Aggregator: aggregator, - } - - return aggregator, nil + Notifier: notifier, + Committee: wrappedCommittee, + Signer: signer, + Persist: persist, + QCCreatedDistributor: qcDistributor, + FinalizationDistributor: finalizationDistributor, + TimeoutCollectorDistributor: timeoutCollectorDistributor, + Forks: forks, + Validator: validator, + VoteAggregator: voteAggregator, + TimeoutAggregator: timeoutAggregator, + } + + return util.MergeReadyDone(voteAggregator, timeoutAggregator), nil }). - Component("consensus compliance engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + Component("consensus participant", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize the block builder var build module.Builder build, err = builder.NewBuilder( @@ -633,15 +657,12 @@ func main() { if err != nil { return nil, fmt.Errorf("could not initialized block builder: %w", err) } - build = blockproducer.NewMetricsWrapper(build, mainMetrics) // wrapper for measuring time spent building block payload component opts := []consensus.Option{ - consensus.WithInitialTimeout(hotstuffTimeout), consensus.WithMinTimeout(hotstuffMinTimeout), - consensus.WithVoteAggregationTimeoutFraction(hotstuffTimeoutVoteAggregationFraction), - consensus.WithTimeoutIncreaseFactor(hotstuffTimeoutIncreaseFactor), - consensus.WithTimeoutDecreaseFactor(hotstuffTimeoutDecreaseFactor), + consensus.WithTimeoutAdjustmentFactor(hotstuffTimeoutAdjustmentFactor), + consensus.WithHappyPathMaxRoundFailures(hotstuffHappyPathMaxRoundFailures), consensus.WithBlockRateDelay(blockRateDelay), consensus.WithConfigRegistrar(node.ConfigManager), } @@ -649,30 +670,50 @@ func main() { if !startupTime.IsZero() { opts = append(opts, consensus.WithStartupTime(startupTime)) } - finalizedBlock, pending, err := recovery.FindLatest(node.State, node.Storage.Headers) if err != nil { return nil, err } + // initialize hotstuff consensus algorithm + hot, err = consensus.NewParticipant( + createLogger(node.Logger, node.RootChainID), + mainMetrics, + build, + finalizedBlock, + pending, + hotstuffModules, + opts..., + ) + if err != nil { + return nil, fmt.Errorf("could not initialize hotstuff engine: %w", err) + } + return hot, nil + }). + Component("consensus compliance engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize the entity database accessors cleaner := bstorage.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) // initialize the pending blocks cache proposals := buffer.NewPendingBlocks() - complianceCore, err := compliance.NewCore(node.Logger, + logger := createLogger(node.Logger, node.RootChainID) + complianceCore, err := compliance.NewCore(logger, node.Metrics.Engine, - node.Tracer, node.Metrics.Mempool, + mainMetrics, node.Metrics.Compliance, + node.Tracer, cleaner, node.Storage.Headers, node.Storage.Payloads, mutableState, proposals, syncCore, - hotstuffModules.Aggregator, + hotstuffModules.Validator, + hot, + hotstuffModules.VoteAggregator, + hotstuffModules.TimeoutAggregator, modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold), ) if err != nil { @@ -681,34 +722,36 @@ func main() { // initialize the compliance engine comp, err = compliance.NewEngine( - node.Logger, - node.Network, + logger, node.Me, - prov, complianceCore, ) if err != nil { return nil, fmt.Errorf("could not initialize compliance engine: %w", err) } - // initialize hotstuff consensus algorithm - hot, err := consensus.NewParticipant( - node.Logger, - mainMetrics, - build, + finalizationDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) + + return comp, nil + }). + Component("consensus message hub", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + messageHub, err := message_hub.NewMessageHub( + createLogger(node.Logger, node.RootChainID), + node.Metrics.Engine, + node.Network, + node.Me, comp, - finalizedBlock, - pending, - hotstuffModules, - opts..., + hot, + hotstuffModules.VoteAggregator, + hotstuffModules.TimeoutAggregator, + node.State, + node.Storage.Payloads, ) if err != nil { - return nil, fmt.Errorf("could not initialize hotstuff engine: %w", err) + return nil, fmt.Errorf("could not create consensus message hub: %w", err) } - - comp = comp.WithConsensus(hot) - finalizationDistributor.AddOnBlockFinalizedConsumer(comp.OnFinalizedBlock) - return comp, nil + hotstuffModules.Notifier.AddConsumer(messageHub) + return messageHub, nil }). Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { finalizedHeader, err = synceng.NewFinalizedHeaderCache(node.Logger, node.State, finalizationDistributor) diff --git a/cmd/consensus/notifier.go b/cmd/consensus/notifier.go index 6b57683186c..94fc57782e6 100644 --- a/cmd/consensus/notifier.go +++ b/cmd/consensus/notifier.go @@ -10,12 +10,19 @@ import ( metricsconsumer "github.com/onflow/flow-go/module/metrics/hotstuff" ) -func createNotifier(log zerolog.Logger, metrics module.HotstuffMetrics, tracer module.Tracer, chain flow.ChainID, -) *pubsub.Distributor { - telemetryConsumer := notifications.NewTelemetryConsumer(log, chain) +// createLogger creates logger which reports chain ID on every log message. +func createLogger(log zerolog.Logger, chainID flow.ChainID) zerolog.Logger { + return log.With().Str("chain", chainID.String()).Logger() +} + +// createNotifier creates a pubsub distributor and connects it to consensus consumers. +func createNotifier(log zerolog.Logger, metrics module.HotstuffMetrics) *pubsub.Distributor { + telemetryConsumer := notifications.NewTelemetryConsumer(log) metricsConsumer := metricsconsumer.NewMetricsConsumer(metrics) + logsConsumer := notifications.NewLogConsumer(log) dis := pubsub.NewDistributor() dis.AddConsumer(telemetryConsumer) dis.AddConsumer(metricsConsumer) + dis.AddConsumer(logsConsumer) return dis } diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 27adcc39e1b..3f12f25edca 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -33,6 +33,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/signature" + validator "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" followereng "github.com/onflow/flow-go/engine/common/follower" @@ -111,7 +112,7 @@ type ExecutionNode struct { collector module.ExecutionMetrics executionState state.ExecutionState followerState protocol.MutableState - committee hotstuff.Committee + committee hotstuff.DynamicCommittee ledgerStorage *ledger.Ledger events *storage.Events serviceEvents *storage.ServiceEvents @@ -124,7 +125,8 @@ type ExecutionNode struct { pendingBlocks *buffer.PendingBlocks // used in follower engine deltas *ingestion.Deltas syncEngine *synchronization.Engine - followerEng *followereng.Engine // to sync blocks from consensus nodes + followerCore *hotstuff.FollowerLoop // follower hotstuff logic + followerEng *followereng.Engine // to sync blocks from consensus nodes computationManager *computation.Manager collectionRequester *requester.Engine ingestionEng *ingestion.Engine @@ -199,6 +201,8 @@ func (builder *ExecutionNodeBuilder) LoadComponentsAndModules() { Component("provider engine", exeNode.LoadProviderEngine). Component("checker engine", exeNode.LoadCheckerEngine). Component("ingestion engine", exeNode.LoadIngestionEngine). + Component("consensus committee", exeNode.LoadConsensusCommittee). + Component("follower core", exeNode.LoadFollowerCore). Component("follower engine", exeNode.LoadFollowerEngine). Component("collection requester engine", exeNode.LoadCollectionRequesterEngine). Component("receipt provider engine", exeNode.LoadReceiptProviderEngine). @@ -258,7 +262,7 @@ func (exeNode *ExecutionNode) LoadExecutionMetrics(node *NodeConfig) error { func (exeNode *ExecutionNode) LoadSyncCore(node *NodeConfig) error { var err error - exeNode.syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + exeNode.syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err } @@ -826,27 +830,34 @@ func (exeNode *ExecutionNode) LoadIngestionEngine( return exeNode.ingestionEng, err } -func (exeNode *ExecutionNode) LoadFollowerEngine( +func (exeNode *ExecutionNode) LoadConsensusCommittee( node *NodeConfig, ) ( module.ReadyDoneAware, error, ) { - // initialize cleaner for DB - cleaner := storage.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) - - // create a finalizer that handles updating the protocol - // state when the follower detects newly finalized blocks - final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, exeNode.followerState, node.Tracer) - // initialize consensus committee's membership state // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee // Note: node.Me.NodeID() is not part of the consensus exeNode.committee - var err error - exeNode.committee, err = committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) if err != nil { return nil, fmt.Errorf("could not create Committee state for main consensus: %w", err) } + node.ProtocolEvents.AddConsumer(committee) + exeNode.committee = committee + + return committee, nil +} + +func (exeNode *ExecutionNode) LoadFollowerCore( + node *NodeConfig, +) ( + module.ReadyDoneAware, + error, +) { + // create a finalizer that handles updating the protocol + // state when the follower detects newly finalized blocks + final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, exeNode.followerState, node.Tracer) packer := signature.NewConsensusSigDataPacker(exeNode.committee) // initialize the verifier for the protocol consensus @@ -862,11 +873,40 @@ func (exeNode *ExecutionNode) LoadFollowerEngine( // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block - followerCore, err := consensus.NewFollower(node.Logger, exeNode.committee, node.Storage.Headers, final, verifier, exeNode.finalizationDistributor, node.RootBlock.Header, node.RootQC, finalized, pending) + exeNode.followerCore, err = consensus.NewFollower( + node.Logger, + exeNode.committee, + node.Storage.Headers, + final, + verifier, + exeNode.finalizationDistributor, + node.RootBlock.Header, + node.RootQC, + finalized, + pending, + ) if err != nil { return nil, fmt.Errorf("could not create follower core logic: %w", err) } + return exeNode.followerCore, nil +} + +func (exeNode *ExecutionNode) LoadFollowerEngine( + node *NodeConfig, +) ( + module.ReadyDoneAware, + error, +) { + // initialize cleaner for DB + cleaner := storage.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) + + packer := signature.NewConsensusSigDataPacker(exeNode.committee) + // initialize the verifier for the protocol consensus + verifier := verification.NewCombinedVerifier(exeNode.committee, packer) + validator := validator.New(exeNode.committee, verifier) + + var err error exeNode.followerEng, err = followereng.New( node.Logger, node.Network, @@ -878,7 +918,8 @@ func (exeNode *ExecutionNode) LoadFollowerEngine( node.Storage.Payloads, exeNode.followerState, exeNode.pendingBlocks, - followerCore, + exeNode.followerCore, + validator, exeNode.syncCore, node.Tracer, followereng.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), diff --git a/cmd/execution_config.go b/cmd/execution_config.go index 19693c239b6..ccc5542420d 100644 --- a/cmd/execution_config.go +++ b/cmd/execution_config.go @@ -13,6 +13,7 @@ import ( exeprovider "github.com/onflow/flow-go/engine/execution/provider" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/engine/execution/computation" "github.com/onflow/flow-go/engine/execution/rpc" @@ -63,6 +64,7 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) { datadir := filepath.Join(homedir, ".flow", "execution") flags.StringVarP(&exeConf.rpcConf.ListenAddr, "rpc-addr", "i", "localhost:9000", "the address the gRPC server listens on") + flags.UintVar(&exeConf.rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") flags.BoolVar(&exeConf.rpcConf.RpcMetricsEnabled, "rpc-metrics-enabled", false, "whether to enable the rpc metrics") flags.StringVar(&exeConf.triedir, "triedir", datadir, "directory to store the execution State") flags.StringVar(&exeConf.executionDataDir, "execution-data-dir", filepath.Join(homedir, ".flow", "execution_data"), "directory to use for storing Execution Data") diff --git a/cmd/ghost/main.go b/cmd/ghost/main.go index 1d7cf6c20dd..d49f11d9aca 100644 --- a/cmd/ghost/main.go +++ b/cmd/ghost/main.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/validator" + "github.com/onflow/flow-go/utils/grpcutils" ) func main() { @@ -18,6 +19,7 @@ func main() { nodeBuilder := cmd.FlowNode("ghost") nodeBuilder.ExtraFlags(func(flags *pflag.FlagSet) { flags.StringVarP(&rpcConf.ListenAddr, "rpc-addr", "r", "localhost:9000", "the address the GRPC server listens on") + flags.UintVar(&rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") }) if err := nodeBuilder.Initialize(); err != nil { diff --git a/cmd/node_builder.go b/cmd/node_builder.go index c94c0d957da..81f5e761452 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -27,6 +27,7 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dns" "github.com/onflow/flow-go/network/p2p/middleware" @@ -130,6 +131,10 @@ type NodeBuilder interface { // ValidateFlags sets any custom validation rules for the command line flags, // for example where certain combinations aren't allowed ValidateFlags(func() error) NodeBuilder + + // ValidateRootSnapshot sets any custom validation rules for the root snapshot. + // This check is executed after other checks but before applying any data from root snapshot. + ValidateRootSnapshot(f func(protocol.Snapshot) error) NodeBuilder } // BaseConfig is the general config for the NodeBuilder and the command line params @@ -198,6 +203,7 @@ type NetworkConfig struct { UnicastMessageTimeout time.Duration DNSCacheTTL time.Duration LibP2PResourceManagerConfig *p2pbuilder.ResourceManagerConfig + ConnectionManagerConfig *connection.ManagerConfig } // NodeConfig contains all the derived parameters such the NodeID, private keys etc. and initialized instances of @@ -221,6 +227,7 @@ type NodeConfig struct { Resolver madns.BasicResolver Middleware network.Middleware Network network.Network + ConduitFactory network.ConduitFactory PingService network.PingService MsgValidators []network.MessageValidator FvmOptions []fvm.Option @@ -249,6 +256,11 @@ type NodeConfig struct { // bootstrapping options SkipNwAddressBasedValidations bool + + // UnicastRateLimiterDistributor notifies consumers when a peer's unicast message is rate limited. + UnicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor + // NodeBlockListDistributor notifies consumers of updates to the node block list + NodeBlockListDistributor *cache.NodeBlockListDistributor } func DefaultBaseConfig() *BaseConfig { @@ -275,6 +287,7 @@ func DefaultBaseConfig() *BaseConfig { UnicastRateLimitDryRun: true, DNSCacheTTL: dns.DefaultTimeToLive, LibP2PResourceManagerConfig: p2pbuilder.DefaultResourceManagerConfig(), + ConnectionManagerConfig: connection.DefaultConnManagerConfig(), }, nodeIDHex: NotSet, AdminAddr: NotSet, @@ -314,7 +327,7 @@ func DefaultBaseConfig() *BaseConfig { } // DependencyList is a slice of ReadyDoneAware implementations that are used by DependableComponent -// to define the list of depenencies that must be ready before starting the component. +// to define the list of dependencies that must be ready before starting the component. type DependencyList struct { components []module.ReadyDoneAware } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 27e32820d5c..1b1da34653e 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -26,6 +26,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" hotsignature "github.com/onflow/flow-go/consensus/hotstuff/signature" + hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" @@ -75,6 +76,7 @@ import ( "github.com/onflow/flow-go/state/protocol/events/gadgets" "github.com/onflow/flow-go/storage" bstorage "github.com/onflow/flow-go/storage/badger" + "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/io" ) @@ -132,6 +134,7 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { MaxHeightRange: backend.DefaultMaxHeightRange, PreferredExecutionNodeIDs: nil, FixedExecutionNodeIDs: nil, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, rpcMetricsEnabled: false, apiRatelimits: nil, @@ -168,10 +171,11 @@ type ObserverServiceBuilder struct { RpcEng *rpc.Engine FinalizationDistributor *pubsub.FinalizationDistributor FinalizedHeader *synceng.FinalizedHeaderCache - Committee hotstuff.Committee + Committee hotstuff.DynamicCommittee Finalized *flow.Header Pending []*flow.Header FollowerCore module.HotStuffFollower + Validator hotstuff.Validator ExecutionDataDownloader execution_data.Downloader ExecutionDataRequester state_synchronization.ExecutionDataRequester // for the observer, the sync engine participants provider is the libp2p peer store which is not // available until after the network has started. Hence, a factory function that needs to be called just before @@ -282,7 +286,7 @@ func (builder *ObserverServiceBuilder) buildFollowerState() *ObserverServiceBuil func (builder *ObserverServiceBuilder) buildSyncCore() *ObserverServiceBuilder { builder.Module("sync core", func(node *cmd.NodeConfig) error { - syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) builder.SyncCore = syncCore return err @@ -292,14 +296,15 @@ func (builder *ObserverServiceBuilder) buildSyncCore() *ObserverServiceBuilder { } func (builder *ObserverServiceBuilder) buildCommittee() *ObserverServiceBuilder { - builder.Module("committee", func(node *cmd.NodeConfig) error { + builder.Component("committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state - // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee + // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS committee // Note: node.Me.NodeID() is not part of the consensus committee committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(committee) builder.Committee = committee - return err + return committee, err }) return builder @@ -325,6 +330,7 @@ func (builder *ObserverServiceBuilder) buildFollowerCore() *ObserverServiceBuild packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) // initialize the verifier for the protocol consensus verifier := verification.NewCombinedVerifier(builder.Committee, packer) + builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) followerCore, err := consensus.NewFollower( node.Logger, @@ -367,6 +373,7 @@ func (builder *ObserverServiceBuilder) buildFollowerEngine() *ObserverServiceBui builder.FollowerState, conCache, builder.FollowerCore, + builder.Validator, builder.SyncCore, node.Tracer, follower.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold)), @@ -576,6 +583,7 @@ func (builder *ObserverServiceBuilder) extraFlags() { flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, "secure-rpc-addr", defaultConfig.rpcConf.SecureGRPCListenAddr, "the address the secure gRPC server listens on") flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") + flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", defaultConfig.rpcConf.MaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") @@ -727,7 +735,8 @@ func (builder *ObserverServiceBuilder) InitIDProviders() { // The following wrapper allows to black-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of blocked nodes to true - builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB) + builder.NodeBlockListDistributor = cache.NewNodeBlockListDistributor() + builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeBlockListDistributor) if err != nil { return fmt.Errorf("could not initialize NodeBlocklistWrapper: %w", err) } @@ -999,7 +1008,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { } // upstream access node forwarder - forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout) + forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, builder.rpcConf.MaxMsgSize) if err != nil { return nil, err } @@ -1032,9 +1041,10 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { // interval, and validators. The network.Middleware is then passed into the initNetwork function. func (builder *ObserverServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, - validators ...network.MessageValidator) network.Middleware { + validators ...network.MessageValidator, +) network.Middleware { slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network) - builder.Middleware = middleware.NewMiddleware( + mw := middleware.NewMiddleware( builder.Logger, libp2pNode, nodeID, builder.Metrics.Bitswap, @@ -1043,8 +1053,10 @@ func (builder *ObserverServiceBuilder) initMiddleware(nodeID flow.Identifier, builder.IDTranslator, builder.CodecFactory(), slashingViolationsConsumer, - middleware.WithMessageValidators(validators...)) - + middleware.WithMessageValidators(validators...), // use default identifier provider + ) + builder.NodeBlockListDistributor.AddConsumer(mw) + builder.Middleware = mw return builder.Middleware } diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 288c5857320..8877440b422 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -49,7 +49,6 @@ import ( "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/network" netcache "github.com/onflow/flow-go/network/cache" - "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/conduit" @@ -123,12 +122,15 @@ type FlowNodeBuilder struct { postShutdownFns []func() error preInitFns []BuilderFunc postInitFns []BuilderFunc + extraRootSnapshotCheck func(protocol.Snapshot) error extraFlagCheck func() error adminCommandBootstrapper *admin.CommandRunnerBootstrapper adminCommands map[string]func(config *NodeConfig) commands.AdminCommand componentBuilder component.ComponentManagerBuilder } +var _ NodeBuilder = (*FlowNodeBuilder)(nil) + func (fnb *FlowNodeBuilder) BaseFlags() { defaultConfig := DefaultBaseConfig() @@ -169,6 +171,11 @@ func (fnb *FlowNodeBuilder) BaseFlags() { fnb.flags.Float64Var(&fnb.BaseConfig.LibP2PResourceManagerConfig.FileDescriptorsRatio, "libp2p-fd-ratio", defaultConfig.LibP2PResourceManagerConfig.FileDescriptorsRatio, "ratio of available file descriptors to be used by libp2p (in (0,1])") fnb.flags.Float64Var(&fnb.BaseConfig.LibP2PResourceManagerConfig.MemoryLimitRatio, "libp2p-memory-limit", defaultConfig.LibP2PResourceManagerConfig.MemoryLimitRatio, "ratio of available memory to be used by libp2p (in (0,1])") + fnb.flags.IntVar(&fnb.BaseConfig.ConnectionManagerConfig.LowWatermark, "libp2p-connmgr-low", defaultConfig.ConnectionManagerConfig.LowWatermark, "low watermarking for libp2p connection manager") + fnb.flags.IntVar(&fnb.BaseConfig.ConnectionManagerConfig.HighWatermark, "libp2p-connmgr-high", defaultConfig.ConnectionManagerConfig.HighWatermark, "high watermarking for libp2p connection manager") + fnb.flags.DurationVar(&fnb.BaseConfig.ConnectionManagerConfig.GracePeriod, "libp2p-connmgr-grace", defaultConfig.ConnectionManagerConfig.GracePeriod, "grace period for libp2p connection manager") + fnb.flags.DurationVar(&fnb.BaseConfig.ConnectionManagerConfig.SilencePeriod, "libp2p-connmgr-silence", defaultConfig.ConnectionManagerConfig.SilencePeriod, "silence period for libp2p connection manager") + fnb.flags.DurationVar(&fnb.BaseConfig.DNSCacheTTL, "dns-cache-ttl", defaultConfig.DNSCacheTTL, "time-to-live for dns cache") fnb.flags.StringSliceVar(&fnb.BaseConfig.PreferredUnicastProtocols, "preferred-unicast-protocols", nil, "preferred unicast protocols in ascending order of preference") fnb.flags.Uint32Var(&fnb.BaseConfig.NetworkReceivedMessageCacheSize, "networking-receive-cache-size", p2p.DefaultReceiveCacheSize, @@ -232,12 +239,12 @@ func (fnb *FlowNodeBuilder) EnqueuePingService() { persist := persister.New(node.DB, node.RootChainID) pingInfoProvider.HotstuffViewFun = func() (uint64, error) { - curView, err := persist.GetStarted() + livenessData, err := persist.GetLivenessData() if err != nil { return 0, err } - return curView, nil + return livenessData.CurrentView, nil } } @@ -281,21 +288,14 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { connGaterInterceptSecureFilters := make([]p2p.PeerFilter, 0) peerManagerFilters := make([]p2p.PeerFilter, 0) - // log and collect metrics for unicast messages that are rate limited - onUnicastRateLimit := func(peerID peer.ID, role, msgType string, topic channels.Topic, reason ratelimit.RateLimitReason) { - fnb.Logger.Warn(). - Str("peer_id", peerID.String()). - Str("role", role). - Str("message_type", msgType). - Str("topic", topic.String()). - Str("reason", reason.String()). - Bool(logging.KeySuspicious, true). - Msg("unicast peer rate limited") - fnb.Metrics.Network.OnRateLimitedUnicastMessage(role, msgType, topic.String(), reason.String()) - } + fnb.UnicastRateLimiterDistributor = ratelimit.NewUnicastRateLimiterDistributor() + fnb.UnicastRateLimiterDistributor.AddConsumer(fnb.Metrics.Network) - // setup default noop unicast rate limiters - unicastRateLimiters := ratelimit.NewRateLimiters(ratelimit.NewNoopRateLimiter(), ratelimit.NewNoopRateLimiter(), onUnicastRateLimit, ratelimit.WithDisabledRateLimiting(fnb.BaseConfig.UnicastRateLimitDryRun)) + // setup default rate limiter options + unicastRateLimiterOpts := []ratelimit.RateLimitersOption{ + ratelimit.WithDisabledRateLimiting(fnb.BaseConfig.UnicastRateLimitDryRun), + ratelimit.WithNotifier(fnb.UnicastRateLimiterDistributor), + } // override noop unicast message rate limiter if fnb.BaseConfig.UnicastMessageRateLimit > 0 { @@ -304,7 +304,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { fnb.BaseConfig.UnicastMessageRateLimit, fnb.BaseConfig.UnicastRateLimitLockoutDuration, ) - unicastRateLimiters.MessageRateLimiter = unicastMessageRateLimiter + unicastRateLimiterOpts = append(unicastRateLimiterOpts, ratelimit.WithMessageRateLimiter(unicastMessageRateLimiter)) // avoid connection gating and pruning during dry run if !fnb.BaseConfig.UnicastRateLimitDryRun { @@ -315,6 +315,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { // don't create outbound connections to rate limited peers peerManagerFilters = append(peerManagerFilters, f) } + } // override noop unicast bandwidth rate limiter @@ -324,7 +325,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { fnb.BaseConfig.UnicastBandwidthBurstLimit, fnb.BaseConfig.UnicastRateLimitLockoutDuration, ) - unicastRateLimiters.BandWidthRateLimiter = unicastBandwidthRateLimiter + unicastRateLimiterOpts = append(unicastRateLimiterOpts, ratelimit.WithBandwidthRateLimiter(unicastBandwidthRateLimiter)) // avoid connection gating and pruning during dry run if !fnb.BaseConfig.UnicastRateLimitDryRun { @@ -335,6 +336,9 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { } } + // setup unicast rate limiters + unicastRateLimiters := ratelimit.NewRateLimiters(unicastRateLimiterOpts...) + fnb.Component(LibP2PNodeComponent, func(node *NodeConfig) (module.ReadyDoneAware, error) { myAddr := fnb.NodeConfig.Me.Address() if fnb.BaseConfig.BindAddr != NotSet { @@ -357,6 +361,7 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { fnb.NetworkConnectionPruning, fnb.PeerUpdateInterval, fnb.LibP2PResourceManagerConfig, + fnb.UnicastRateLimiterDistributor, ) libp2pNode, err := libP2PNodeFactory() @@ -406,8 +411,7 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, } slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(fnb.Logger, fnb.Metrics.Network) - - fnb.Middleware = middleware.NewMiddleware( + mw := middleware.NewMiddleware( fnb.Logger, fnb.LibP2PNode, fnb.Me.NodeID(), @@ -418,6 +422,8 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory(node *NodeConfig, fnb.CodecFactory(), slashingViolationsConsumer, mwOpts...) + fnb.NodeBlockListDistributor.AddConsumer(mw) + fnb.Middleware = mw subscriptionManager := subscription.NewChannelSubscriptionManager(fnb.Middleware) var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() @@ -550,6 +556,11 @@ func (fnb *FlowNodeBuilder) ParseAndPrintFlags() error { return fnb.extraFlagsValidation() } +func (fnb *FlowNodeBuilder) ValidateRootSnapshot(f func(protocol.Snapshot) error) NodeBuilder { + fnb.extraRootSnapshotCheck = f + return fnb +} + func (fnb *FlowNodeBuilder) ValidateFlags(f func() error) NodeBuilder { fnb.extraFlagCheck = f return fnb @@ -954,7 +965,8 @@ func (fnb *FlowNodeBuilder) InitIDProviders() { // The following wrapper allows to black-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of blocked nodes to true - blocklistWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB) + fnb.NodeBlockListDistributor = cache.NewNodeBlockListDistributor() + blocklistWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB, fnb.NodeBlockListDistributor) if err != nil { return fmt.Errorf("could not initialize NodeBlocklistWrapper: %w", err) } @@ -1096,6 +1108,14 @@ func (fnb *FlowNodeBuilder) setRootSnapshot(rootSnapshot protocol.Snapshot) erro return fmt.Errorf("failed to validate root snapshot QCs: %w", err) } + // perform extra checks requested by specific node types + if fnb.extraRootSnapshotCheck != nil { + err = fnb.extraRootSnapshotCheck(rootSnapshot) + if err != nil { + return fmt.Errorf("failed to perform extra checks on root snapshot: %w", err) + } + } + fnb.RootSnapshot = rootSnapshot // cache properties of the root snapshot, for convenience fnb.RootResult, fnb.RootSeal, err = fnb.RootSnapshot.SealedResult() @@ -1478,25 +1498,6 @@ func (fnb *FlowNodeBuilder) OverrideComponent(name string, f ReadyDoneFactory) N return fnb.Component(name, f) } -// OverrideModule adds given builder function to the modules set of the node builder. If a builder function with that name -// already exists, it will be overridden. -func (fnb *FlowNodeBuilder) OverrideModule(name string, f BuilderFunc) NodeBuilder { - for i := 0; i < len(fnb.modules); i++ { - if fnb.modules[i].name == name { - // found module with the name, override it. - fnb.modules[i] = namedModuleFunc{ - fn: f, - name: name, - } - - return fnb - } - } - - // no module found with the same name, hence just adding it. - return fnb.Module(name, f) -} - // RestartableComponent adds a new component to the node that conforms to the ReadyDoneAware // interface, and calls the provided error handler when an irrecoverable error is encountered. // Use RestartableComponent if the component is not critical to the node's safe operation and @@ -1519,6 +1520,25 @@ func (fnb *FlowNodeBuilder) RestartableComponent(name string, f ReadyDoneFactory return fnb } +// OverrideModule adds given builder function to the modules set of the node builder. If a builder function with that name +// already exists, it will be overridden. +func (fnb *FlowNodeBuilder) OverrideModule(name string, f BuilderFunc) NodeBuilder { + for i := 0; i < len(fnb.modules); i++ { + if fnb.modules[i].name == name { + // found module with the name, override it. + fnb.modules[i] = namedModuleFunc{ + fn: f, + name: name, + } + + return fnb + } + } + + // no module found with the same name, hence just adding it. + return fnb.Module(name, f) +} + func (fnb *FlowNodeBuilder) PreInit(f BuilderFunc) NodeBuilder { fnb.preInitFns = append(fnb.preInitFns, f) return fnb diff --git a/cmd/testclient/go.mod b/cmd/testclient/go.mod index 2d60a4a36e6..0a02e69ad42 100644 --- a/cmd/testclient/go.mod +++ b/cmd/testclient/go.mod @@ -5,14 +5,14 @@ go 1.19 require ( github.com/onflow/flow-go-sdk v0.4.1 github.com/spf13/pflag v1.0.5 - google.golang.org/grpc v1.28.0 + google.golang.org/grpc v1.52.3 ) require ( github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/ethereum/go-ethereum v1.9.9 // indirect - github.com/golang/protobuf v1.3.5 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/onflow/cadence v0.4.0 // indirect github.com/onflow/flow/protobuf/go/flow v0.1.5-0.20200601215056-34a11def1d6b // indirect github.com/pkg/errors v0.8.1 // indirect @@ -22,9 +22,10 @@ require ( github.com/segmentio/fasthash v1.0.2 // indirect github.com/stretchr/testify v1.5.1 // indirect golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect - golang.org/x/text v0.3.2 // indirect - google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.2.4 // indirect ) diff --git a/cmd/testclient/go.sum b/cmd/testclient/go.sum index fc2a9f66b8c..dd3500c37d1 100644 --- a/cmd/testclient/go.sum +++ b/cmd/testclient/go.sum @@ -69,13 +69,16 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -189,8 +192,9 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -206,11 +210,13 @@ golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,13 +229,19 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/cmd/testclient/main.go b/cmd/testclient/main.go index e381837b65c..ce42309c78e 100644 --- a/cmd/testclient/main.go +++ b/cmd/testclient/main.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" sdk "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/client" @@ -26,7 +27,7 @@ func main() { pflag.Parse() - c, err := client.New(targetAddr, grpc.WithInsecure()) //nolint:staticcheck + c, err := client.New(targetAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { panic(err) } @@ -43,12 +44,15 @@ func main() { panic(err) } - account := sdk.NewAccountKey(). + accountKey := sdk.NewAccountKey(). FromPrivateKey(sk). SetHashAlgo(crypto.SHA3_256). SetWeight(sdk.AccountKeyWeightThreshold) - signer := crypto.NewInMemorySigner(sk, account.HashAlgo) + signer, err := crypto.NewInMemorySigner(sk, accountKey.HashAlgo) + if err != nil { + panic(err) + } addr := sdk.NewAddressGenerator(sdk.Testnet).NextAddress() @@ -77,7 +81,7 @@ func main() { } `)). SetGasLimit(100). - SetProposalKey(addr, account.ID, nonce). + SetProposalKey(addr, accountKey.Index, nonce). SetReferenceBlockID(latest.ID). SetPayer(addr). AddAuthorizer(addr) diff --git a/cmd/util/cmd/checkpoint-collect-stats/cmd.go b/cmd/util/cmd/checkpoint-collect-stats/cmd.go index 2c1595d21e9..cf74b467758 100644 --- a/cmd/util/cmd/checkpoint-collect-stats/cmd.go +++ b/cmd/util/cmd/checkpoint-collect-stats/cmd.go @@ -2,11 +2,11 @@ package checkpoint_collect_stats import ( "bufio" - "bytes" "encoding/json" "math" "os" "path/filepath" + "strings" "github.com/montanaflynn/stats" "github.com/pkg/profile" @@ -17,11 +17,11 @@ import ( "github.com/onflow/atree" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/ledger/complete" "github.com/onflow/flow-go/ledger/complete/wal" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/debug" ) @@ -227,9 +227,9 @@ func getType(key ledger.Key) string { return "account's cadence public domain map" case "contract": return "account's cadence contract domain map" - case state.ContractNamesKey: + case flow.ContractNamesKey: return "contract names" - case state.AccountStatusKey: + case flow.AccountStatusKey: return "account status" case "uuid": return "uuid generator state" @@ -237,10 +237,10 @@ func getType(key ledger.Key) string { return "address generator state" } // other fvm registers - if bytes.HasPrefix(k, []byte("public_key_")) { + if strings.HasPrefix(kstr, "public_key_") { return "public key" } - if bytes.HasPrefix(k, []byte(state.CodeKeyPrefix)) { + if strings.HasPrefix(kstr, flow.CodeKeyPrefix) { return "contract content" } return "others" diff --git a/cmd/util/cmd/common/flow_client.go b/cmd/util/cmd/common/flow_client.go index dc02b33cbc2..e16438da9f6 100644 --- a/cmd/util/cmd/common/flow_client.go +++ b/cmd/util/cmd/common/flow_client.go @@ -5,8 +5,10 @@ import ( "strings" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" client "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/flow/order" @@ -62,7 +64,10 @@ func secureFlowClient(accessAddress, accessApiNodePubKey string) (*client.Client } // create flow client - flowClient, err := client.NewClient(accessAddress, dialOpts) + flowClient, err := client.NewClient(accessAddress, + dialOpts, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + ) if err != nil { return nil, err } @@ -73,7 +78,10 @@ func secureFlowClient(accessAddress, accessApiNodePubKey string) (*client.Client // insecureFlowClient creates flow client with insecure GRPC connection func insecureFlowClient(accessAddress string) (*client.Client, error) { // create flow client - flowClient, err := client.NewClient(accessAddress, grpc.WithInsecure()) //nolint:staticcheck + flowClient, err := client.NewClient(accessAddress, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + ) if err != nil { return nil, fmt.Errorf("failed to create flow client %w", err) } diff --git a/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go b/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go index 8d44bb8b8f8..f615d56943a 100644 --- a/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go +++ b/cmd/util/cmd/exec-data-json-export/delta_snapshot_exporter.go @@ -59,7 +59,7 @@ func ExportDeltaSnapshots(blockID flow.Identifier, dbPath string, outputPath str // end of snapshots return nil } - m, err := snap[0].Delta.MarshalJSON() + m, err := json.Marshal(snap[0].Delta.UpdatedRegisters()) if err != nil { return fmt.Errorf("could not load delta snapshot: %w", err) } diff --git a/cmd/util/cmd/hotstuff-view/cmd/getter.go b/cmd/util/cmd/hotstuff-view/cmd/getter.go deleted file mode 100644 index f366f1b3c10..00000000000 --- a/cmd/util/cmd/hotstuff-view/cmd/getter.go +++ /dev/null @@ -1,48 +0,0 @@ -package cmd - -import ( - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - - "github.com/onflow/flow-go/cmd/util/cmd/common" - "github.com/onflow/flow-go/consensus/hotstuff/persister" -) - -var GetterCmd = &cobra.Command{ - Use: "get", - Short: "get hotstuff view", - Run: runGet, -} - -func init() { - rootCmd.AddCommand(GetterCmd) -} - -func runGet(*cobra.Command, []string) { - db := common.InitStorage(flagDatadir) - defer db.Close() - - storages := common.InitStorages(db) - state, err := common.InitProtocolState(db, storages) - if err != nil { - log.Fatal().Err(err).Msg("could not init protocol state") - } - - rootBlock, err := state.Params().Root() - if err != nil { - log.Fatal().Err(err).Msgf("could not get root block") - } - - pers := persister.New(db, rootBlock.ChainID) - - reader := NewReader(pers) - - log.Info().Msg("getting hotstuff view") - - view, err := reader.GetHotstuffView() - if err != nil { - log.Fatal().Err(err).Msg("could not get hotstuff view") - } - - log.Info().Msgf("successfully get hotstuff view: %v", view) -} diff --git a/cmd/util/cmd/hotstuff-view/cmd/reader.go b/cmd/util/cmd/hotstuff-view/cmd/reader.go deleted file mode 100644 index 83d4ca3cac9..00000000000 --- a/cmd/util/cmd/hotstuff-view/cmd/reader.go +++ /dev/null @@ -1,48 +0,0 @@ -package cmd - -import ( - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff/persister" -) - -type Reader struct { - persister *persister.Persister -} - -func NewReader(persister *persister.Persister) *Reader { - return &Reader{ - persister: persister, - } -} - -func (r *Reader) SetHotstuffView(view uint64) error { - if view == 0 { - return fmt.Errorf("hotstuff view is not allowed to set to 0, please specify --view") - } - - // ensure we don't set the view lower than the current view, because this may cause this node to violate protocol rules - currentView, err := r.GetHotstuffView() - if err != nil { - return err - } - if view < currentView { - return fmt.Errorf("hotstuff view %v cannot be lower than current view (%d)", view, currentView) - } - - err = r.persister.PutStarted(view) - if err != nil { - return fmt.Errorf("could not put hotstuff view %v: %w", view, err) - } - - return nil -} - -func (r *Reader) GetHotstuffView() (uint64, error) { - view, err := r.persister.GetStarted() - if err != nil { - return 0, fmt.Errorf("could not get hotstuff view %w", err) - } - - return view, nil -} diff --git a/cmd/util/cmd/hotstuff-view/cmd/setter.go b/cmd/util/cmd/hotstuff-view/cmd/setter.go deleted file mode 100644 index 24fe9c90714..00000000000 --- a/cmd/util/cmd/hotstuff-view/cmd/setter.go +++ /dev/null @@ -1,59 +0,0 @@ -package cmd - -import ( - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - - "github.com/onflow/flow-go/cmd/util/cmd/common" - "github.com/onflow/flow-go/consensus/hotstuff/persister" -) - -var ( - flagView uint64 -) - -// TODO: will be removed after active pacemaker is implemented -var SetterCmd = &cobra.Command{ - Use: "set", - Short: "set hotstuff view", - Run: runSet, -} - -func init() { - rootCmd.AddCommand(SetterCmd) - SetterCmd.Flags().Uint64Var(&flagView, "view", 0, "hotstuff view") -} - -func runSet(*cobra.Command, []string) { - db := common.InitStorage(flagDatadir) - defer db.Close() - - storages := common.InitStorages(db) - state, err := common.InitProtocolState(db, storages) - if err != nil { - log.Fatal().Err(err).Msg("could not init protocol state") - } - - rootBlock, err := state.Params().Root() - if err != nil { - log.Fatal().Err(err).Msgf("could not get root block") - } - - pers := persister.New(db, rootBlock.ChainID) - - reader := NewReader(pers) - - currentView, err := reader.GetHotstuffView() - if err != nil { - log.Fatal().Err(err).Msgf("could not get current view") - } - - log.Info().Msgf("current view: %v, setting hotstuff view to %v", currentView, flagView) - - err = reader.SetHotstuffView(flagView) - if err != nil { - log.Fatal().Err(err).Msgf("could not set hotstuff view to %v", flagView) - } - - log.Info().Msgf("successfully set hotstuff view to %v", flagView) -} diff --git a/cmd/util/cmd/hotstuff-view/main.go b/cmd/util/cmd/hotstuff-view/main.go deleted file mode 100644 index 71b88409020..00000000000 --- a/cmd/util/cmd/hotstuff-view/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/onflow/flow-go/cmd/util/cmd/hotstuff-view/cmd" - -func main() { - cmd.Execute() -} diff --git a/cmd/util/cmd/read-execution-state/list-accounts/cmd.go b/cmd/util/cmd/read-execution-state/list-accounts/cmd.go index 8e1c10b2bee..181e54180c6 100644 --- a/cmd/util/cmd/read-execution-state/list-accounts/cmd.go +++ b/cmd/util/cmd/read-execution-state/list-accounts/cmd.go @@ -75,28 +75,31 @@ func run(*cobra.Command, []string) { log.Fatal().Err(err).Msgf("invalid chain name") } - ldg := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - - ledgerKey := executionState.RegisterIDToKey(flow.NewRegisterID(owner, key)) - path, err := pathfinder.KeyToPath(ledgerKey, complete.DefaultPathFinderVersion) - if err != nil { - log.Fatal().Err(err).Msgf("cannot convert key to path") - } - - read := &ledger.TrieRead{ - RootHash: ledger.RootHash(stateCommitment), - Paths: []ledger.Path{ - path, - }, - } - - values, err := forest.Read(read) - if err != nil { - return nil, err - } - - return values[0], nil - }) + ldg := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + + ledgerKey := executionState.RegisterIDToKey(id) + path, err := pathfinder.KeyToPath( + ledgerKey, + complete.DefaultPathFinderVersion) + if err != nil { + log.Fatal().Err(err).Msgf("cannot convert key to path") + } + + read := &ledger.TrieRead{ + RootHash: ledger.RootHash(stateCommitment), + Paths: []ledger.Path{ + path, + }, + } + + values, err := forest.Read(read) + if err != nil { + return nil, err + } + + return values[0], nil + })) txnState := state.NewTransactionState(ldg, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) diff --git a/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go b/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go new file mode 100644 index 00000000000..c6eb12e2c43 --- /dev/null +++ b/cmd/util/cmd/read-hotstuff/cmd/get_liveness.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/cmd/util/cmd/common" +) + +var GetLivenessCmd = &cobra.Command{ + Use: "get-liveness", + Short: "get hotstuff liveness data (current view, newest QC, last view TC)", + Run: runGetLivenessData, +} + +func init() { + rootCmd.AddCommand(GetLivenessCmd) +} + +func runGetLivenessData(*cobra.Command, []string) { + db := common.InitStorage(flagDatadir) + defer db.Close() + + storages := common.InitStorages(db) + state, err := common.InitProtocolState(db, storages) + if err != nil { + log.Fatal().Err(err).Msg("could not init protocol state") + } + + rootBlock, err := state.Params().Root() + if err != nil { + log.Fatal().Err(err).Msgf("could not get root block") + } + + reader := NewHotstuffReader(db, rootBlock.ChainID) + + log.Info().Msg("getting hotstuff liveness data") + + livenessData, err := reader.GetLivenessData() + if err != nil { + log.Fatal().Err(err).Msg("could not get hotstuff liveness data") + } + + log.Info().Msgf("successfully get hotstuff liveness data") + common.PrettyPrint(livenessData) +} diff --git a/cmd/util/cmd/read-hotstuff/cmd/get_safety.go b/cmd/util/cmd/read-hotstuff/cmd/get_safety.go new file mode 100644 index 00000000000..bd0281990c7 --- /dev/null +++ b/cmd/util/cmd/read-hotstuff/cmd/get_safety.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/cmd/util/cmd/common" +) + +var GetSafetyCmd = &cobra.Command{ + Use: "get-safety", + Short: "get hotstuff safety data (locked view, highest acked view, last timeout)", + Run: runGetSafetyData, +} + +func init() { + rootCmd.AddCommand(GetSafetyCmd) +} + +func runGetSafetyData(*cobra.Command, []string) { + db := common.InitStorage(flagDatadir) + defer db.Close() + + storages := common.InitStorages(db) + state, err := common.InitProtocolState(db, storages) + if err != nil { + log.Fatal().Err(err).Msg("could not init protocol state") + } + + rootBlock, err := state.Params().Root() + if err != nil { + log.Fatal().Err(err).Msgf("could not get root block") + } + + reader := NewHotstuffReader(db, rootBlock.ChainID) + + log.Info().Msg("getting hotstuff safety data") + + livenessData, err := reader.GetSafetyData() + if err != nil { + log.Fatal().Err(err).Msg("could not get hotstuff safety data") + } + + log.Info().Msgf("successfully get hotstuff safety data") + common.PrettyPrint(livenessData) +} diff --git a/cmd/util/cmd/read-hotstuff/cmd/reader.go b/cmd/util/cmd/read-hotstuff/cmd/reader.go new file mode 100644 index 00000000000..dcfefe94f7a --- /dev/null +++ b/cmd/util/cmd/read-hotstuff/cmd/reader.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/persister" + "github.com/onflow/flow-go/model/flow" +) + +// HotstuffReader exposes only the read-only parts of the Hotstuff Persister component. +// This is used to read information about the HotStuff instance's current state from CLI tools. +// CAUTION: the write functions are hidden here, because it is NOT SAFE to use them outside +// the Hotstuff state machine. +type HotstuffReader interface { + // GetLivenessData retrieves the latest persisted liveness data. + GetLivenessData() (*hotstuff.LivenessData, error) + // GetSafetyData retrieves the latest persisted safety data. + GetSafetyData() (*hotstuff.SafetyData, error) +} + +// NewHotstuffReader returns a new Persister, constrained to read-only operations. +func NewHotstuffReader(db *badger.DB, chainID flow.ChainID) HotstuffReader { + return persister.New(db, chainID) +} diff --git a/cmd/util/cmd/hotstuff-view/cmd/root.go b/cmd/util/cmd/read-hotstuff/cmd/root.go similarity index 88% rename from cmd/util/cmd/hotstuff-view/cmd/root.go rename to cmd/util/cmd/read-hotstuff/cmd/root.go index 8f75b9a69a4..57dfcba65a6 100644 --- a/cmd/util/cmd/hotstuff-view/cmd/root.go +++ b/cmd/util/cmd/read-hotstuff/cmd/root.go @@ -13,8 +13,8 @@ var ( ) var rootCmd = &cobra.Command{ - Use: "hotstuff-view", - Short: "get or set hotstuff current view", + Use: "read-hotstuff", + Short: "read hotstuff liveness/safety data", } var RootCmd = rootCmd diff --git a/cmd/util/cmd/read-hotstuff/main.go b/cmd/util/cmd/read-hotstuff/main.go new file mode 100644 index 00000000000..2308b66b341 --- /dev/null +++ b/cmd/util/cmd/read-hotstuff/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/onflow/flow-go/cmd/util/cmd/read-hotstuff/cmd" + +func main() { + cmd.Execute() +} diff --git a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go index 5854233e15a..d4e36002742 100644 --- a/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go +++ b/cmd/util/cmd/rollback-executed-height/cmd/rollback_executed_height_test.go @@ -7,6 +7,7 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/bootstrap" "github.com/onflow/flow-go/model/flow" @@ -64,7 +65,10 @@ func TestReExecuteBlock(t *testing.T) { require.NotNil(t, es) // prepare data - header := unittest.BlockHeaderWithParentFixture(genesis) // make sure the height is higher than genesis + executableBlock := unittest.ExecutableBlockFixtureWithParent( + nil, + genesis) // make sure the height is higher than genesis + header := executableBlock.Block.Header executionReceipt := unittest.ExecutionReceiptFixture() executionReceipt.ExecutionResult.BlockID = header.ID() cdp := make([]*flow.ChunkDataPack, 0, len(executionReceipt.ExecutionResult.Chunks)) @@ -81,16 +85,20 @@ func TestReExecuteBlock(t *testing.T) { err = headers.Store(header) require.NoError(t, err) + computationResult := &execution.ComputationResult{ + ExecutableBlock: executableBlock, + EndState: endState, + ChunkDataPacks: cdp, + Events: []flow.EventsList{blockEvents.Events}, + ServiceEvents: se.Events, + TransactionResults: tes, + } + // save execution results err = es.SaveExecutionResults( context.Background(), - header, - endState, - cdp, + computationResult, executionReceipt, - []flow.EventsList{blockEvents.Events}, - se.Events, - tes, ) require.NoError(t, err) @@ -155,13 +163,8 @@ func TestReExecuteBlock(t *testing.T) { // re execute result err = es.SaveExecutionResults( context.Background(), - header, - endState, - cdp, + computationResult, executionReceipt, - []flow.EventsList{blockEvents.Events}, - se.Events, - tes, ) require.NoError(t, err) @@ -216,7 +219,10 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { require.NotNil(t, es) // prepare data - header := unittest.BlockHeaderWithParentFixture(genesis) // make sure the height is higher than genesis + executableBlock := unittest.ExecutableBlockFixtureWithParent( + nil, + genesis) // make sure the height is higher than genesis + header := executableBlock.Block.Header executionReceipt := unittest.ExecutionReceiptFixture() executionReceipt.ExecutionResult.BlockID = header.ID() cdp := make([]*flow.ChunkDataPack, 0, len(executionReceipt.ExecutionResult.Chunks)) @@ -233,16 +239,20 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { err = headers.Store(header) require.NoError(t, err) + computationResult := &execution.ComputationResult{ + ExecutableBlock: executableBlock, + EndState: endState, + ChunkDataPacks: cdp, + Events: []flow.EventsList{blockEvents.Events}, + ServiceEvents: se.Events, + TransactionResults: tes, + } + // save execution results err = es.SaveExecutionResults( context.Background(), - header, - endState, - cdp, + computationResult, executionReceipt, - []flow.EventsList{blockEvents.Events}, - se.Events, - tes, ) require.NoError(t, err) @@ -298,16 +308,20 @@ func TestReExecuteBlockWithDifferentResult(t *testing.T) { endState2, err := executionReceipt2.ExecutionResult.FinalStateCommitment() require.NoError(t, err) + computationResult2 := &execution.ComputationResult{ + ExecutableBlock: executableBlock, + EndState: endState2, + ChunkDataPacks: cdp2, + Events: []flow.EventsList{blockEvents.Events}, + ServiceEvents: se.Events, + TransactionResults: tes, + } + // re execute result err = es.SaveExecutionResults( context.Background(), - header, - endState2, - cdp2, + computationResult2, executionReceipt2, - []flow.EventsList{blockEvents.Events}, - se.Events, - tes, ) require.NoError(t, err) diff --git a/cmd/util/cmd/root.go b/cmd/util/cmd/root.go index 4ee583d941c..c2700d80bbd 100644 --- a/cmd/util/cmd/root.go +++ b/cmd/util/cmd/root.go @@ -19,6 +19,7 @@ import ( export_json_transactions "github.com/onflow/flow-go/cmd/util/cmd/export-json-transactions" read_badger "github.com/onflow/flow-go/cmd/util/cmd/read-badger/cmd" read_execution_state "github.com/onflow/flow-go/cmd/util/cmd/read-execution-state" + read_hotstuff "github.com/onflow/flow-go/cmd/util/cmd/read-hotstuff/cmd" read_protocol_state "github.com/onflow/flow-go/cmd/util/cmd/read-protocol-state/cmd" index_er "github.com/onflow/flow-go/cmd/util/cmd/reindex/cmd" rollback_executed_height "github.com/onflow/flow-go/cmd/util/cmd/rollback-executed-height/cmd" @@ -74,6 +75,7 @@ func addCommands() { rootCmd.AddCommand(read_execution_state.Cmd) rootCmd.AddCommand(snapshot.Cmd) rootCmd.AddCommand(export_json_transactions.Cmd) + rootCmd.AddCommand(read_hotstuff.RootCmd) } func initConfig() { diff --git a/cmd/util/ledger/migrations/account_migration.go b/cmd/util/ledger/migrations/account_migration.go index b3fad0e9ece..51c123712f1 100644 --- a/cmd/util/ledger/migrations/account_migration.go +++ b/cmd/util/ledger/migrations/account_migration.go @@ -6,8 +6,8 @@ import ( "github.com/rs/zerolog/log" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" ) func MigrateAccountUsage(payloads []ledger.Payload, nWorker int) ([]ledger.Payload, error) { @@ -24,7 +24,7 @@ func payloadSize(key ledger.Key, payload ledger.Payload) (uint64, error) { } func isAccountKey(key ledger.Key) bool { - return string(key.KeyParts[1].Value) == state.AccountStatusKey + return string(key.KeyParts[1].Value) == flow.AccountStatusKey } type AccountUsageMigrator struct{} diff --git a/cmd/util/ledger/migrations/storage_fees_migration.go b/cmd/util/ledger/migrations/storage_fees_migration.go index dec4b251c66..d55a725d90b 100644 --- a/cmd/util/ledger/migrations/storage_fees_migration.go +++ b/cmd/util/ledger/migrations/storage_fees_migration.go @@ -23,17 +23,14 @@ func StorageFeesMigration(payload []ledger.Payload) ([]ledger.Payload, error) { for s, u := range storageUsed { // this is the storage used by the storage_used register we are about to add - storageUsedByStorageUsed := fvm.RegisterSize( - flow.BytesToAddress([]byte(s)), - "storage_used", - make([]byte, 8)) + id := flow.NewRegisterID( + string(flow.BytesToAddress([]byte(s)).Bytes()), + "storage_used") + storageUsedByStorageUsed := fvm.RegisterSize(id, make([]byte, 8)) u = u + uint64(storageUsedByStorageUsed) newPayload = append(newPayload, *ledger.NewPayload( - registerIDToKey(flow.RegisterID{ - Owner: s, - Key: "storage_used", - }), + registerIDToKey(id), utils.Uint64ToBinary(u), )) } @@ -61,7 +58,5 @@ func incrementStorageUsed(p ledger.Payload, used map[string]uint64) error { } func registerSize(id flow.RegisterID, p ledger.Payload) int { - address := flow.BytesToAddress([]byte(id.Owner)) - key := id.Key - return fvm.RegisterSize(address, key, p.Value()) + return fvm.RegisterSize(id, p.Value()) } diff --git a/cmd/util/ledger/migrations/utils.go b/cmd/util/ledger/migrations/utils.go index 4644e833e44..506efe61db0 100644 --- a/cmd/util/ledger/migrations/utils.go +++ b/cmd/util/ledger/migrations/utils.go @@ -51,9 +51,9 @@ var _ atree.Ledger = &AccountsAtreeLedger{} func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { v, err := a.Accounts.GetValue( - flow.BytesToAddress(owner), - string(key), - ) + flow.NewRegisterID( + string(flow.BytesToAddress(owner).Bytes()), + string(key))) if err != nil { return nil, fmt.Errorf("getting value failed: %w", err) } @@ -62,10 +62,10 @@ func (a *AccountsAtreeLedger) GetValue(owner, key []byte) ([]byte, error) { func (a *AccountsAtreeLedger) SetValue(owner, key, value []byte) error { err := a.Accounts.SetValue( - flow.BytesToAddress(owner), - string(key), - value, - ) + flow.NewRegisterID( + string(flow.BytesToAddress(owner).Bytes()), + string(key)), + value) if err != nil { return fmt.Errorf("setting value failed: %w", err) } diff --git a/cmd/util/ledger/reporters/atree_reporter.go b/cmd/util/ledger/reporters/atree_reporter.go index 9210a17b8e2..6d1be625125 100644 --- a/cmd/util/ledger/reporters/atree_reporter.go +++ b/cmd/util/ledger/reporters/atree_reporter.go @@ -8,12 +8,11 @@ import ( "sync" "github.com/onflow/atree" - - fvmState "github.com/onflow/flow-go/fvm/state" - "github.com/onflow/flow-go/ledger" - "github.com/rs/zerolog" "github.com/schollz/progressbar/v3" + + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" ) // AtreeReporter iterates payloads and generates payload and atree stats. @@ -116,12 +115,14 @@ func getPayloadType(p *ledger.Payload) (payloadType, error) { if len(k.KeyParts) < 2 { return unknownPayloadType, nil } - if fvmState.IsFVMStateKey( + + id := flow.NewRegisterID( string(k.KeyParts[0].Value), - string(k.KeyParts[1].Value), - ) { + string(k.KeyParts[1].Value)) + if id.IsInternalState() { return fvmPayloadType, nil } + if bytes.HasPrefix(k.KeyParts[1].Value, []byte(atree.LedgerBaseStorageSlabPrefix)) { return slabPayloadType, nil } diff --git a/cmd/verification_builder.go b/cmd/verification_builder.go index f5384312200..6b1a007d848 100644 --- a/cmd/verification_builder.go +++ b/cmd/verification_builder.go @@ -7,9 +7,11 @@ import ( "github.com/spf13/pflag" flowconsensus "github.com/onflow/flow-go/consensus" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" hotsignature "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recoveryprotocol "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/engine/common/follower" @@ -102,8 +104,10 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { finalizationDistributor *pubsub.FinalizationDistributor finalizedHeader *commonsync.FinalizedHeaderCache - followerEng *follower.Engine // the follower engine - collector module.VerificationMetrics // used to collect metrics of all engines + committee *committees.Consensus + followerCore *hotstuff.FollowerLoop // follower hotstuff logic + followerEng *follower.Engine // the follower engine + collector module.VerificationMetrics // used to collect metrics of all engines ) v.FlowNodeBuilder. @@ -189,7 +193,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { Module("sync core", func(node *NodeConfig) error { var err error - syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + syncCore, err = chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) return err }). Component("verifier engine", func(node *NodeConfig) (module.ReadyDoneAware, error) { @@ -317,22 +321,21 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { return blockConsumer, nil }). - Component("follower engine", func(node *NodeConfig) (module.ReadyDoneAware, error) { - // initialize cleaner for DB - cleaner := badger.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) + Component("consensus committee", func(node *NodeConfig) (module.ReadyDoneAware, error) { + // initialize consensus committee's membership state + // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee + // Note: node.Me.NodeID() is not part of the consensus committee + var err error + committee, err = committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(committee) + return committee, err + }). + Component("follower core", func(node *NodeConfig) (module.ReadyDoneAware, error) { // create a finalizer that handles updating the protocol // state when the follower detects newly finalized blocks final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, followerState, node.Tracer) - // initialize consensus committee's membership state - // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee - // Note: node.Me.NodeID() is not part of the consensus committee - committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) - if err != nil { - return nil, fmt.Errorf("could not create Committee state for main consensus: %w", err) - } - packer := hotsignature.NewConsensusSigDataPacker(committee) // initialize the verifier for the protocol consensus verifier := verification.NewCombinedVerifier(committee, packer) @@ -347,12 +350,34 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { // creates a consensus follower with ingestEngine as the notifier // so that it gets notified upon each new finalized block - followerCore, err := flowconsensus.NewFollower(node.Logger, committee, node.Storage.Headers, final, verifier, finalizationDistributor, node.RootBlock.Header, - node.RootQC, finalized, pending) + followerCore, err = flowconsensus.NewFollower( + node.Logger, + committee, + node.Storage.Headers, + final, + verifier, + finalizationDistributor, + node.RootBlock.Header, + node.RootQC, + finalized, + pending, + ) if err != nil { return nil, fmt.Errorf("could not create follower core logic: %w", err) } + return followerCore, nil + }). + Component("follower engine", func(node *NodeConfig) (module.ReadyDoneAware, error) { + // initialize cleaner for DB + cleaner := badger.NewCleaner(node.Logger, node.DB, node.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency) + + packer := hotsignature.NewConsensusSigDataPacker(committee) + // initialize the verifier for the protocol consensus + verifier := verification.NewCombinedVerifier(committee, packer) + validator := validator.New(committee, verifier) + + var err error followerEng, err = follower.New( node.Logger, node.Network, @@ -365,6 +390,7 @@ func (v *VerificationNodeBuilder) LoadComponentsAndModules() { followerState, pendingBlocks, followerCore, + validator, syncCore, node.Tracer, follower.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)), diff --git a/consensus/aggregators.go b/consensus/aggregators.go new file mode 100644 index 00000000000..10bf86083c8 --- /dev/null +++ b/consensus/aggregators.go @@ -0,0 +1,83 @@ +package consensus + +import ( + "fmt" + + "github.com/gammazero/workerpool" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutaggregator" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutcollector" + "github.com/onflow/flow-go/consensus/hotstuff/voteaggregator" + "github.com/onflow/flow-go/consensus/hotstuff/votecollector" + "github.com/onflow/flow-go/module" +) + +// NewVoteAggregator creates new VoteAggregator and subscribes for finalization events. +// No error returns are expected during normal operations. +func NewVoteAggregator( + log zerolog.Logger, + hotstuffMetrics module.HotstuffMetrics, + engineMetrics module.EngineMetrics, + mempoolMetrics module.MempoolMetrics, + lowestRetainedView uint64, + notifier hotstuff.Consumer, + voteProcessorFactory hotstuff.VoteProcessorFactory, + distributor *pubsub.FinalizationDistributor, +) (hotstuff.VoteAggregator, error) { + + createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) + voteCollectors := voteaggregator.NewVoteCollectors(log, lowestRetainedView, workerpool.New(4), createCollectorFactoryMethod) + + // initialize the vote aggregator + aggregator, err := voteaggregator.NewVoteAggregator( + log, + hotstuffMetrics, + engineMetrics, + mempoolMetrics, + notifier, + lowestRetainedView, + voteCollectors, + ) + if err != nil { + return nil, fmt.Errorf("could not create vote aggregator: %w", err) + } + distributor.AddOnBlockFinalizedConsumer(aggregator.OnFinalizedBlock) + + return aggregator, nil +} + +// NewTimeoutAggregator creates new TimeoutAggregator and connects Hotstuff event source with event handler. +// No error returns are expected during normal operations. +func NewTimeoutAggregator(log zerolog.Logger, + hotstuffMetrics module.HotstuffMetrics, + engineMetrics module.EngineMetrics, + mempoolMetrics module.MempoolMetrics, + notifier *pubsub.Distributor, + timeoutProcessorFactory hotstuff.TimeoutProcessorFactory, + distributor *pubsub.TimeoutCollectorDistributor, + lowestRetainedView uint64, +) (hotstuff.TimeoutAggregator, error) { + + timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory(log, notifier, distributor, timeoutProcessorFactory) + collectors := timeoutaggregator.NewTimeoutCollectors(log, lowestRetainedView, timeoutCollectorFactory) + + // initialize the timeout aggregator + aggregator, err := timeoutaggregator.NewTimeoutAggregator( + log, + hotstuffMetrics, + engineMetrics, + mempoolMetrics, + notifier, + lowestRetainedView, + collectors, + ) + if err != nil { + return nil, fmt.Errorf("could not create timeout aggregator: %w", err) + } + notifier.AddConsumer(aggregator) + + return aggregator, nil +} diff --git a/consensus/config.go b/consensus/config.go index b2db429d71a..8862ffd366e 100644 --- a/consensus/config.go +++ b/consensus/config.go @@ -12,38 +12,40 @@ import ( // HotstuffModules is a helper structure to encapsulate dependencies to create // a hotStuff participant. type HotstuffModules struct { - Notifier hotstuff.Consumer // observer for hotstuff events - Committee hotstuff.Committee // consensus committee - Signer hotstuff.Signer // signer of proposal & votes - Persist hotstuff.Persister // last state of consensus participant - FinalizationDistributor *pubsub.FinalizationDistributor // observer for finalization events, used by compliance engine - QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader - Forks hotstuff.Forks // information about multiple forks - Validator hotstuff.Validator // validator of proposals & votes - Aggregator hotstuff.VoteAggregator // aggregator of votes, used by leader + Committee hotstuff.DynamicCommittee // consensus committee + Signer hotstuff.Signer // signer of proposal & votes + Persist hotstuff.Persister // last state of consensus participant + Notifier *pubsub.Distributor // observer for hotstuff events + FinalizationDistributor *pubsub.FinalizationDistributor // observer for finalization events, used by compliance engine + QCCreatedDistributor *pubsub.QCCreatedDistributor // observer for qc created event, used by leader + TimeoutCollectorDistributor *pubsub.TimeoutCollectorDistributor + Forks hotstuff.Forks // information about multiple forks + Validator hotstuff.Validator // validator of proposals & votes + VoteAggregator hotstuff.VoteAggregator // aggregator of votes, used by leader + TimeoutAggregator hotstuff.TimeoutAggregator // aggregator of `TimeoutObject`s, used by every replica } type ParticipantConfig struct { - StartupTime time.Time // the time when consensus participant enters first view - TimeoutInitial time.Duration // the initial timeout for the pacemaker - TimeoutMinimum time.Duration // the minimum timeout for the pacemaker - TimeoutAggregationFraction float64 // the percentage part of the timeout period reserved for vote aggregation - TimeoutIncreaseFactor float64 // the factor at which the timeout grows when timeouts occur - TimeoutDecreaseFactor float64 // the factor at which the timeout grows when timeouts occur - BlockRateDelay time.Duration // a delay to broadcast block proposal in order to control the block production rate - Registrar updatable_configs.Registrar // optional: for registering HotStuff configs as dynamically configurable + StartupTime time.Time // the time when consensus participant enters first view + TimeoutMinimum time.Duration // the minimum timeout for the pacemaker + TimeoutMaximum time.Duration // the maximum timeout for the pacemaker + TimeoutAdjustmentFactor float64 // the factor at which the timeout duration is adjusted + HappyPathMaxRoundFailures uint64 // number of failed rounds before first timeout increase + BlockRateDelay time.Duration // a delay to broadcast block proposal in order to control the block production rate + MaxTimeoutObjectRebroadcastInterval time.Duration // maximum interval for timeout object rebroadcast + Registrar updatable_configs.Registrar // optional: for registering HotStuff configs as dynamically configurable } func DefaultParticipantConfig() ParticipantConfig { defTimeout := timeout.DefaultConfig cfg := ParticipantConfig{ - TimeoutInitial: time.Duration(defTimeout.ReplicaTimeout) * time.Millisecond, - TimeoutMinimum: time.Duration(defTimeout.MinReplicaTimeout) * time.Millisecond, - TimeoutAggregationFraction: defTimeout.VoteAggregationTimeoutFraction, - TimeoutIncreaseFactor: defTimeout.TimeoutIncrease, - TimeoutDecreaseFactor: defTimeout.TimeoutDecrease, - BlockRateDelay: defTimeout.GetBlockRateDelay(), - Registrar: nil, + TimeoutMinimum: time.Duration(defTimeout.MinReplicaTimeout) * time.Millisecond, + TimeoutMaximum: time.Duration(defTimeout.MaxReplicaTimeout) * time.Millisecond, + TimeoutAdjustmentFactor: defTimeout.TimeoutAdjustmentFactor, + HappyPathMaxRoundFailures: defTimeout.HappyPathMaxRoundFailures, + BlockRateDelay: defTimeout.GetBlockRateDelay(), + MaxTimeoutObjectRebroadcastInterval: time.Duration(defTimeout.MaxTimeoutObjectRebroadcastInterval) * time.Millisecond, + Registrar: nil, } return cfg } @@ -56,33 +58,21 @@ func WithStartupTime(time time.Time) Option { } } -func WithInitialTimeout(timeout time.Duration) Option { - return func(cfg *ParticipantConfig) { - cfg.TimeoutInitial = timeout - } -} - func WithMinTimeout(timeout time.Duration) Option { return func(cfg *ParticipantConfig) { cfg.TimeoutMinimum = timeout } } -func WithTimeoutIncreaseFactor(factor float64) Option { - return func(cfg *ParticipantConfig) { - cfg.TimeoutIncreaseFactor = factor - } -} - -func WithTimeoutDecreaseFactor(factor float64) Option { +func WithTimeoutAdjustmentFactor(factor float64) Option { return func(cfg *ParticipantConfig) { - cfg.TimeoutDecreaseFactor = factor + cfg.TimeoutAdjustmentFactor = factor } } -func WithVoteAggregationTimeoutFraction(fraction float64) Option { +func WithHappyPathMaxRoundFailures(happyPathMaxRoundFailures uint64) Option { return func(cfg *ParticipantConfig) { - cfg.TimeoutAggregationFraction = fraction + cfg.HappyPathMaxRoundFailures = happyPathMaxRoundFailures } } diff --git a/consensus/follower.go b/consensus/follower.go index 9ce96c50325..bcae0a08fb7 100644 --- a/consensus/follower.go +++ b/consensus/follower.go @@ -16,26 +16,26 @@ import ( // TODO: this needs to be integrated with proper configuration and bootstrapping. -func NewFollower(log zerolog.Logger, committee hotstuff.Committee, headers storage.Headers, updater module.Finalizer, +func NewFollower(log zerolog.Logger, committee hotstuff.DynamicCommittee, headers storage.Headers, updater module.Finalizer, verifier hotstuff.Verifier, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate, finalized *flow.Header, pending []*flow.Header) (*hotstuff.FollowerLoop, error) { - finalizer, err := newFinalizer(finalized, headers, updater, notifier, rootHeader, rootQC) + forks, err := NewForks(finalized, headers, updater, notifier, rootHeader, rootQC) if err != nil { - return nil, fmt.Errorf("could not initialize finalizer: %w", err) + return nil, fmt.Errorf("could not initialize forks: %w", err) } // initialize the Validator - validator := validator.New(committee, finalizer, verifier) + validator := validator.New(committee, verifier) // recover the hotstuff state as a follower - err = recovery.Follower(log, finalizer, validator, finalized, pending) + err = recovery.Follower(log, forks, validator, finalized, pending) if err != nil { return nil, fmt.Errorf("could not recover hotstuff follower state: %w", err) } // initialize the follower logic - logic, err := follower.New(log, validator, finalizer) + logic, err := follower.New(log, validator, forks) if err != nil { return nil, fmt.Errorf("could not create follower logic: %w", err) } diff --git a/consensus/follower_test.go b/consensus/follower_test.go index 06b36e97f19..26a61c88ae5 100644 --- a/consensus/follower_test.go +++ b/consensus/follower_test.go @@ -1,6 +1,7 @@ package consensus_test import ( + "context" "os" "testing" "time" @@ -14,9 +15,11 @@ import ( "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" mockmodule "github.com/onflow/flow-go/module/mock" mockstorage "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" @@ -49,18 +52,21 @@ func TestHotStuffFollower(t *testing.T) { type HotStuffFollowerSuite struct { suite.Suite - committee *mockhotstuff.Committee - headers *mockstorage.Headers - updater *mockmodule.Finalizer - verifier *mockhotstuff.Verifier - notifier *mockhotstuff.FinalizationConsumer - rootHeader *flow.Header - rootQC *flow.QuorumCertificate - finalized *flow.Header - pending []*flow.Header - follower *hotstuff.FollowerLoop - + committee *mockhotstuff.DynamicCommittee + headers *mockstorage.Headers + finalizer *mockmodule.Finalizer + verifier *mockhotstuff.Verifier + notifier *mockhotstuff.FinalizationConsumer + rootHeader *flow.Header + rootQC *flow.QuorumCertificate + finalized *flow.Header + pending []*flow.Header + follower *hotstuff.FollowerLoop mockConsensus *MockConsensus + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error } // SetupTest initializes all the components needed for the Follower. @@ -70,43 +76,47 @@ func (s *HotStuffFollowerSuite) SetupTest() { s.mockConsensus = &MockConsensus{identities: identities} // mock consensus committee - s.committee = &mockhotstuff.Committee{} - s.committee.On("Identities", mock.Anything).Return( - func(blockID flow.Identifier) flow.IdentityList { + s.committee = &mockhotstuff.DynamicCommittee{} + s.committee.On("IdentitiesByEpoch", mock.Anything).Return( + func(_ uint64) flow.IdentityList { return identities }, nil, ) for _, identity := range identities { - s.committee.On("Identity", mock.Anything, identity.NodeID).Return(identity, nil) + s.committee.On("IdentityByEpoch", mock.Anything, identity.NodeID).Return(identity, nil) + s.committee.On("IdentityByBlock", mock.Anything, identity.NodeID).Return(identity, nil) } s.committee.On("LeaderForView", mock.Anything).Return( func(view uint64) flow.Identifier { return identities[int(view)%len(identities)].NodeID }, nil, ) + s.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(identities.TotalWeight()), nil) // mock storage headers s.headers = &mockstorage.Headers{} - // mock finalization updater - s.updater = &mockmodule.Finalizer{} + // mock finalization finalizer + s.finalizer = mockmodule.NewFinalizer(s.T()) - // mock finalization updater - s.verifier = &mockhotstuff.Verifier{} - s.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything).Return(nil) - s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything).Return(nil) + // mock finalization finalizer + s.verifier = mockhotstuff.NewVerifier(s.T()) + s.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + s.verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() // mock consumer for finalization notifications - s.notifier = &mockhotstuff.FinalizationConsumer{} + s.notifier = mockhotstuff.NewFinalizationConsumer(s.T()) // root block and QC parentID, err := flow.HexStringToIdentifier("aa7693d498e9a087b1cadf5bfe9a1ff07829badc1915c210e482f369f9a00a70") require.NoError(s.T(), err) s.rootHeader = &flow.Header{ - ParentID: parentID, - Timestamp: time.Now().UTC(), - Height: 21053, - View: 52078, + ParentID: parentID, + Timestamp: time.Now().UTC(), + Height: 21053, + View: 52078, + ParentView: 52077, } signerIndices, err := signature.EncodeSignersToIndices(identities.NodeIDs(), identities.NodeIDs()[:3]) @@ -125,14 +135,12 @@ func (s *HotStuffFollowerSuite) SetupTest() { // BeforeTest instantiates and starts Follower func (s *HotStuffFollowerSuite) BeforeTest(suiteName, testName string) { - s.notifier.On("OnBlockIncorporated", blockWithID(s.rootHeader.ID())).Return().Once() - var err error s.follower, err = consensus.NewFollower( zerolog.New(os.Stderr), s.committee, s.headers, - s.updater, + s.finalizer, s.verifier, s.notifier, s.rootHeader, @@ -142,106 +150,90 @@ func (s *HotStuffFollowerSuite) BeforeTest(suiteName, testName string) { ) require.NoError(s.T(), err) - select { - case <-s.follower.Ready(): - case <-time.After(time.Second): - s.T().Error("timeout on waiting for follower start") - } + s.ctx, s.cancel, s.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + s.follower.Start(s.ctx) + unittest.RequireCloseBefore(s.T(), s.follower.Ready(), time.Second, "follower failed to start") } -// AfterTest stops follower and asserts that the Follower executed the expected callbacks -// to s.updater.MakeValid or s.notifier.OnBlockIncorporated +// AfterTest stops follower and asserts that the Follower executed the expected callbacks. func (s *HotStuffFollowerSuite) AfterTest(suiteName, testName string) { + s.cancel() + unittest.RequireCloseBefore(s.T(), s.follower.Done(), time.Second, "follower failed to stop") select { - case <-s.follower.Done(): - case <-time.After(time.Second): - s.T().Error("timeout on waiting for expected Follower shutdown") - s.T().FailNow() // stops the test + case err := <-s.errs: + require.NoError(s.T(), err) + default: } - s.notifier.AssertExpectations(s.T()) - s.updater.AssertExpectations(s.T()) } // TestInitialization verifies that the basic test setup with initialization of the Follower works as expected func (s *HotStuffFollowerSuite) TestInitialization() { - // we expect no additional calls to s.updater or s.notifier besides what is already specified in BeforeTest + // we expect no additional calls to s.finalizer or s.notifier besides what is already specified in BeforeTest } // TestSubmitProposal verifies that when submitting a single valid block (child's root block), -// the Follower reacts with callbacks to s.updater.MakeValid and s.notifier.OnBlockIncorporated with this new block +// the Follower reacts with callbacks to s.notifier.OnBlockIncorporated with this new block func (s *HotStuffFollowerSuite) TestSubmitProposal() { rootBlockView := s.rootHeader.View nextBlock := s.mockConsensus.extendBlock(rootBlockView+1, s.rootHeader) s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.updater.On("MakeValid", blockID(nextBlock.ID())).Return(nil).Once() - s.submitWithTimeout(nextBlock, rootBlockView) + s.submitProposal(nextBlock) } -// TestFollowerFinalizedBlock verifies that when submitting 4 extra blocks -// the Follower reacts with callbacks to s.updater.MakeValid or s.notifier.OnBlockIncorporated +// TestFollowerFinalizedBlock verifies that when submitting 2 extra blocks +// the Follower reacts with callbacks to s.notifier.OnBlockIncorporated // for all the added blocks. Furthermore, the follower should finalize the first submitted block, -// i.e. call s.updater.MakeFinal and s.notifier.OnFinalizedBlock +// i.e. call s.finalizer.MakeFinal and s.notifier.OnFinalizedBlock func (s *HotStuffFollowerSuite) TestFollowerFinalizedBlock() { expectedFinalized := s.mockConsensus.extendBlock(s.rootHeader.View+1, s.rootHeader) s.notifier.On("OnBlockIncorporated", blockWithID(expectedFinalized.ID())).Return().Once() - s.updater.On("MakeValid", blockID(expectedFinalized.ID())).Return(nil).Once() - s.submitWithTimeout(expectedFinalized, s.rootHeader.View) + s.submitProposal(expectedFinalized) // direct 1-chain on top of expectedFinalized nextBlock := s.mockConsensus.extendBlock(expectedFinalized.View+1, expectedFinalized) s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.updater.On("MakeValid", blockID(nextBlock.ID())).Return(nil).Once() - s.submitWithTimeout(nextBlock, expectedFinalized.View) + s.submitProposal(nextBlock) - // direct 2-chain on top of expectedFinalized - lastBlock := nextBlock - nextBlock = s.mockConsensus.extendBlock(lastBlock.View+1, lastBlock) - s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.updater.On("MakeValid", blockID(nextBlock.ID())).Return(nil).Once() - s.submitWithTimeout(nextBlock, lastBlock.View) + done := make(chan struct{}) - // indirect 3-chain on top of expectedFinalized => finalization - lastBlock = nextBlock + // indirect 2-chain on top of expectedFinalized + lastBlock := nextBlock nextBlock = s.mockConsensus.extendBlock(lastBlock.View+5, lastBlock) - s.notifier.On("OnFinalizedBlock", blockWithID(expectedFinalized.ID())).Return().Once() s.notifier.On("OnBlockIncorporated", blockWithID(nextBlock.ID())).Return().Once() - s.updater.On("MakeFinal", blockID(expectedFinalized.ID())).Return(nil).Once() - s.updater.On("MakeValid", blockID(nextBlock.ID())).Return(nil).Once() - s.submitWithTimeout(nextBlock, lastBlock.View) + s.notifier.On("OnFinalizedBlock", blockWithID(expectedFinalized.ID())).Return().Once() + s.finalizer.On("MakeFinal", blockID(expectedFinalized.ID())).Run(func(_ mock.Arguments) { + close(done) + }).Return(nil).Once() + s.submitProposal(nextBlock) + unittest.RequireCloseBefore(s.T(), done, time.Second, "expect to close before timeout") } // TestOutOfOrderBlocks verifies that when submitting a variety of blocks with view numbers -// OUT OF ORDER, the Follower reacts with callbacks to s.updater.MakeValid or s.notifier.OnBlockIncorporated +// OUT OF ORDER, the Follower reacts with callbacks to s.notifier.OnBlockIncorporated // for all the added blocks. Furthermore, we construct the test such that the follower should finalize // eventually a bunch of blocks in one go. // The following illustrates the tree of submitted blocks, with notation // -// - [a, b] is a block at view "b" with a QC with view "a", -// e.g., [1, 2] means a block at view "2" with an included QC for view "1" -// -// . [52078+15, 52078+20] (should finalize this fork) -// . | -// . | -// . [52078+14, 52078+15] -// . | -// . | -// . [52078+13, 52078+14] -// . | -// . | -// . [52078+11, 52078+12] [52078+11, 52078+17] [52078+ 9, 52078+13] [52078+ 9, 52078+10] -// . \ | | / -// . \| | / -// . [52078+ 7, 52078+ 8] [52078+ 7, 52078+11] [52078+ 5, 52078+ 9] [52078+ 5, 52078+ 6] -// . \ | | / -// . \| | / -// . [52078+ 3, 52078+ 4] [52078+ 3, 52078+ 7] [52078+ 1, 52078+ 5] [52078+ 1, 52078+ 2] -// . \ | | / -// . \| | / -// . [52078+ 0, 52078+ 3] [52078+ 0, 52078+ 1] -// . \ / -// . \ / -// . [52078+ 0, x] (root block; no qc to parent) +// [52078+14, 52078+20] (should finalize this fork) +// | +// | +// [52078+13, 52078+14] +// | +// | +// [52078+11, 52078+17] [52078+ 9, 52078+13] [52078+ 9, 52078+10] +// | | / +// | | / +// [52078+ 7, 52078+ 8] [52078+ 7, 52078+11] [52078+ 5, 52078+ 9] [52078+ 5, 52078+ 6] +// \ | | / +// \| | / +// [52078+ 3, 52078+ 4] [52078+ 3, 52078+ 7] [52078+ 1, 52078+ 5] [52078+ 1, 52078+ 2] +// \ | | / +// \| | / +// [52078+ 0, 52078+ 3] [52078+ 0, 52078+ 1] +// \ / +// \ / +// [52078+ 0, x] (root block; no qc to parent) func (s *HotStuffFollowerSuite) TestOutOfOrderBlocks() { // in the following, we reference the block's by their view minus the view of the // root block (52078). E.g. block [52078+ 9, 52078+10] would be referenced as `block10` @@ -261,49 +253,49 @@ func (s *HotStuffFollowerSuite) TestOutOfOrderBlocks() { block09 := s.mockConsensus.extendBlock(rootView+9, block05) block06 := s.mockConsensus.extendBlock(rootView+6, block05) - block12 := s.mockConsensus.extendBlock(rootView+12, block11) block17 := s.mockConsensus.extendBlock(rootView+17, block11) block13 := s.mockConsensus.extendBlock(rootView+13, block09) block10 := s.mockConsensus.extendBlock(rootView+10, block09) block14 := s.mockConsensus.extendBlock(rootView+14, block13) - block15 := s.mockConsensus.extendBlock(rootView+15, block14) - block20 := s.mockConsensus.extendBlock(rootView+20, block15) + block20 := s.mockConsensus.extendBlock(rootView+20, block14) - for _, b := range []*flow.Header{block01, block02, block03, block04, block05, block06, block07, block08, block09, block10, block11, block12, block13, block14, block15, block17, block20} { + for _, b := range []*flow.Header{block01, block02, block03, block04, block05, block06, block07, block08, block09, block10, block11, block13, block14, block17, block20} { s.notifier.On("OnBlockIncorporated", blockWithID(b.ID())).Return().Once() - s.updater.On("MakeValid", blockID(b.ID())).Return(nil).Once() } // now we feed the blocks in some wild view order into the Follower // (Caution: we still have to make sure the parent is known, before we give its child to the Follower) - s.submitWithTimeout(block03, rootView) - s.submitWithTimeout(block07, rootView+3) - s.submitWithTimeout(block11, rootView+7) - s.submitWithTimeout(block01, rootView) - s.submitWithTimeout(block12, rootView+11) - s.submitWithTimeout(block05, rootView+1) - s.submitWithTimeout(block17, rootView+11) - s.submitWithTimeout(block09, rootView+5) - s.submitWithTimeout(block06, rootView+5) - s.submitWithTimeout(block10, rootView+9) - s.submitWithTimeout(block04, rootView+3) - s.submitWithTimeout(block13, rootView+9) - s.submitWithTimeout(block14, rootView+13) - s.submitWithTimeout(block08, rootView+7) - s.submitWithTimeout(block15, rootView+14) - s.submitWithTimeout(block02, rootView+1) + s.submitProposal(block03) + s.submitProposal(block07) + s.submitProposal(block11) + s.submitProposal(block01) + s.submitProposal(block05) + s.submitProposal(block17) + s.submitProposal(block09) + s.submitProposal(block06) + s.submitProposal(block10) + s.submitProposal(block04) + s.submitProposal(block13) + s.submitProposal(block14) + s.submitProposal(block08) + s.submitProposal(block02) + + done := make(chan struct{}) // Block 20 should now finalize the fork up to and including block13 s.notifier.On("OnFinalizedBlock", blockWithID(block01.ID())).Return().Once() - s.updater.On("MakeFinal", blockID(block01.ID())).Return(nil).Once() + s.finalizer.On("MakeFinal", blockID(block01.ID())).Return(nil).Once() s.notifier.On("OnFinalizedBlock", blockWithID(block05.ID())).Return().Once() - s.updater.On("MakeFinal", blockID(block05.ID())).Return(nil).Once() + s.finalizer.On("MakeFinal", blockID(block05.ID())).Return(nil).Once() s.notifier.On("OnFinalizedBlock", blockWithID(block09.ID())).Return().Once() - s.updater.On("MakeFinal", blockID(block09.ID())).Return(nil).Once() + s.finalizer.On("MakeFinal", blockID(block09.ID())).Return(nil).Once() s.notifier.On("OnFinalizedBlock", blockWithID(block13.ID())).Return().Once() - s.updater.On("MakeFinal", blockID(block13.ID())).Return(nil).Once() - s.submitWithTimeout(block20, rootView+15) + s.finalizer.On("MakeFinal", blockID(block13.ID())).Run(func(_ mock.Arguments) { + close(done) + }).Return(nil).Once() + s.submitProposal(block20) + unittest.RequireCloseBefore(s.T(), done, time.Second, "expect to close before timeout") } // blockWithID returns a testify `argumentMatcher` that only accepts blocks with the given ID @@ -316,21 +308,9 @@ func blockID(expectedBlockID flow.Identifier) interface{} { return mock.MatchedBy(func(blockID flow.Identifier) bool { return expectedBlockID == blockID }) } -// submitWithTimeout submits the given (proposal, parentView) pair to the Follower. As the follower -// might block on this call, we add a timeout that fails the test, in case of a dead-lock. -func (s *HotStuffFollowerSuite) submitWithTimeout(proposal *flow.Header, parentView uint64) { - sent := make(chan struct{}) - go func() { - s.follower.SubmitProposal(proposal, parentView) - close(sent) - }() - select { - case <-sent: - case <-time.After(time.Second): - s.T().Error("timeout on waiting for expected Follower shutdown") - s.T().FailNow() // stops the test - } - +// submitProposal submits the given (proposal, parentView) pair to the Follower. +func (s *HotStuffFollowerSuite) submitProposal(proposal *flow.Header) { + s.follower.SubmitProposal(model.ProposalFromFlow(proposal)) } // MockConsensus is used to generate Blocks for a mocked consensus committee @@ -344,5 +324,20 @@ func (mc *MockConsensus) extendBlock(blockView uint64, parent *flow.Header) *flo nextBlock.ProposerID = mc.identities[int(blockView)%len(mc.identities)].NodeID signerIndices, _ := signature.EncodeSignersToIndices(mc.identities.NodeIDs(), mc.identities.NodeIDs()) nextBlock.ParentVoterIndices = signerIndices + if nextBlock.View == parent.View+1 { + nextBlock.LastViewTC = nil + } else { + newestQC := unittest.QuorumCertificateFixture(func(qc *flow.QuorumCertificate) { + qc.View = parent.View + qc.SignerIndices = signerIndices + }) + nextBlock.LastViewTC = &flow.TimeoutCertificate{ + View: blockView - 1, + NewestQCViews: []uint64{newestQC.View}, + NewestQC: newestQC, + SignerIndices: signerIndices, + SigData: unittest.SignatureFixture(), + } + } return nextBlock } diff --git a/consensus/hotstuff/README.md b/consensus/hotstuff/README.md index e495cdfd8b1..c2ede0b97c9 100644 --- a/consensus/hotstuff/README.md +++ b/consensus/hotstuff/README.md @@ -1,66 +1,138 @@ # Flow's HotStuff We use a BFT consensus algorithm with deterministic finality in Flow for -* Consensus Nodes: deciding on the content of blocks -* Cluster of Collector Nodes: batching transactions into collections. - -Flow uses a derivative of HotStuff -(see paper [HotStuff: BFT Consensus in the Lens of Blockchain (version 6)](https://arxiv.org/abs/1803.05069v6) for the original algorithm). -Our implementation is a mixture of Chained HotStuff and Event-Driven HotStuff with a few additional, minor modifications: -* We employ the rules for locking on blocks and block finalization from Event-Driven HotStuff. -Specifically, we lock the newest block which has an (direct or indirect) 2-chain built on top. -We finalize a block when it has a 3-chain built on top where with the first two chain links being _direct_. -* The way we progress though views follows the Chained HotStuff algorithm. -A replica only votes or proposes for its current view. -* Flow's HotStuff does not implement `ViewChange` messages -(akin to the approach taken in [Streamlet: Textbook Streamlined Blockchains](https://eprint.iacr.org/2020/088.pdf)) -* Flow uses a decentralized random beacon (based on [Dfinity's proposal](https://dfinity.org/pdf-viewer/library/dfinity-consensus.pdf)). -The random beacon is run by Flow's consensus nodes and integrated into the consensus voting process. - -#### Node weights - -In Flow, there are multiple HotStuff instances running in parallel. Specifically, the consensus nodes form a HotStuff committee and each collector cluster is its own committee. In the following, we refer to an authorized set of nodes, who run a particular HotStuff instance as a (HotStuff) `committee`. +* Consensus Nodes: decide on the content of blocks by including collections of transactions, +* Cluster of Collector Nodes: batch transactions into collections. + +Conceptually, Flow uses a derivative of [HotStuff](https://arxiv.org/abs/1803.05069) called Jolteon. +It was originally described in the paper ['Jolteon and Ditto: Network-Adaptive Efficient Consensus with Asynchronous Fallback'](https://arxiv.org/abs/2106.10362), +published June 2021 by Meta’s blockchain research team Novi and academic collaborators. +Meta’s team (then called 'Diem') [implemented](https://github.com/diem/diem/blob/latest/consensus/README.md) Jolteon with marginal modifications and named it +[DiemBFT v4](https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf), +which was subsequently rebranded as [AptosBFT](https://github.com/aptos-labs/aptos-core/tree/main/developer-docs-site/static/papers/aptos-consensus-state-machine-replication-in-the-aptos-blockchain). +Conceptually, Jolteon (DiemBFT v4, AptosBFT) belongs to the family of HotStuff consensus protocols, +but adds two significant improvements over the original HotStuff: (i) Jolteon incorporates a PaceMaker with active message exchange for view synchronization +and (ii) utilizes the most efficient 2-chain commit rule. + +The foundational innovation in the original HotStuff was its pipelining of block production and finalization. +It utilizes leaders for information collection and to drive consensus, which makes it highly message-efficient. +In HotStuff, the consensus mechanics are cleverly arranged such that the protocol runs as fast as network conditions permit. +(Not be limited by fixed, minimal wait times for messages.) +This property is called "responsiveness" and is very important in practise. +HotStuff is a round-based consensus algorithm, which requires a supermajority of nodes to be in the same view to make progress. It is the role +of the pacemaker to guarantee that eventually a supermajority of nodes will be in the same view. In the original HotStuff, the pacemaker was +essentially left as a black box. The only requirement was that the pacemaker had to get the nodes eventually into the same view. +Vanilla HotStuff requires 3 subsequent children to finalize a block on the happy path (aka '3-chain rule'). +In the original HotStuff paper, the authors discuss the more efficient 2-chain rule. They explain a timing-related edge-case, where the protocol +could theoretically get stuck in a timeout loop without progress. To guarantee liveness despite this edge case, the event-driven HotStuff variant in the original paper employs +the 3-chain rule. + +As this discussion illustrates, HotStuff is more a family of algorithms: the pacemaker is conceptually separated and can be implemented +in may ways. The finality rule is easily swapped out. In addition, there are various other degrees of freedom left open in the family +of HotStuff protocols. +The Jolteon protocol extends HotStuff by specifying one particular pacemaker, which utilizes dedicated messages to synchronize views and provides very strong guarantees. +Thereby, the Jolteon pacemaker closes the timing edge case forcing the original HotStuff to use the 3-chain rule for liveness. +As a consequence, the Jolteon protocol can utilize the most efficient 2-chain rule. +While Jolteon's close integration of the pacemaker into the consensus meachanics changes the correctness and liveness proofs significantly, +the protocol's runtime behaviour matches the HotStuff framework. Therefore, we categorize Jolteon, DiemBFT v4, and AptosBFT as members of the HotStuff family. + +Flow's consensus is largely an implementation of Jolteon with some elements from DiemBFT v4. While these consensus protocols are identical on a conceptual level, +they subtly differ in the information included into the different consensus messages. For Flow's implementation, we combined nuances +from Jolteon and DiemBFT v4 as follows to improve runtime efficiency, reduce code complexity, and minimize the surface for byzantine attacks: -* Flow allows nodes to have different weights, reflecting how much they are trusted by the protocol. - The weight of a node can change over time due to stake changes or discovering protocol violations. - A `super-majority` of nodes is defined as a subset of the consensus committee, - where the nodes have _more_ than 2/3 of the entire committee's accumulated weight. -* The messages from zero-weighted nodes are ignored by all committee members. +* Flow's `TimeoutObject` implements the `timeout` message in Jolteon. + * In the Jolteon protocol, + the `timeout` message contains the view `V`, which the sender wishes to abandon and the latest + Quorum Certificate [QC] known to the sender. Due to successive leader failures, the QC might + not be from the previous view, i.e. `QC.View + 1 < V` is possible. When receiving a `timeout` message, + it is possible for the recipient to advance to round `QC.View + 1`, + but not necessarily to view `V`, as a malicious sender might set `V` to an erroneously large value. + On the one hand, a recipient that has fallen behind cannot catch up to view `V` immediately. On the other hand, the recipient must + cache the timeout to guarantee liveness, making it vulnerable to memory exhaustion attacks. + * [DiemBFT v4](https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf) introduced the additional rule that the `timeout` + must additionally include the Timeout Certificate [TC] for the previous view, + if and only if the contained QC is _not_ from the previous round (i.e. `QC.View + 1 < V`). Conceptually, + this means that the sender of the timeout message must prove that they entered round `V` according to protocol rules. + In other words, malicious nodes cannot send timeouts for future views that they should not have entered. + + For Flow, we follow the convention from DiemBFT v4. This modification simplifies byzantine-resilient processing of `TimeoutObject`s, + avoiding subtle spamming and memory exhaustion attacks. Furthermore, it speeds up the recovery of crashed nodes. + +* For consensus votes, we stick with the original Jolteon format, i.e. we do _not_ include the highest QC known to the voter, which is the case in DiemBFT v4. + The QC is useful only on the unhappy path, where a node has missed some recent blocks. However, including a QC in every vote adds consistent overhead to the happy + path. In Jolteon as well as DiemBFT v4, the timeout messages already contain the highest known QCs. Therefore, the highest QC is already + shared among the network in the unhappy path even without including it in the votes. +* In Jolteon, the TC contains the full QCs from a supermajority of nodes, which have some overhead in size. DiemBFT v4 improves this by only including + the QC's respective views in the TC. Flow utilizes this optimization from DiemBFT v4. + +In the following, we will use the terms Jolteon and HotStuff interchangeably to refer to Flow's consensus implementation. +Beyond the Realm of HotStuff and Jolteon, we have added the following advancement to Flow's consensus system: + +* Flow contains a **decentralized random beacon** (based on [Dfinity's proposal](https://dfinity.org/pdf-viewer/library/dfinity-consensus.pdf)). + The random beacon is run by Flow's consensus nodes and integrated into the consensus voting process. The random beacon provides a nearly + unbiasable source of entropy natively within the protocol that is verifiable and deterministic. The random beacon can be used to generate pseudo random + numbers, which we use within Flow protocol in various places. We plan to also use the random beacon to implement secure pseudo random number generators in Candence. + ## Architecture -#### Determining block validity +_Concepts and Terminology_ + +In Flow, there are multiple HotStuff instances running in parallel. Specifically, the consensus nodes form a HotStuff committee and each collector cluster is its own committee. +In the following, we refer to an authorized set of nodes, who run a particular HotStuff instance as a (HotStuff) `committee`. + +* Flow allows nodes to have different weights, reflecting how much they are trusted by the protocol. + The weight of a node can change over time due to stake changes or discovering protocol violations. + A `super-majority` of nodes is defined as a subset of the consensus committee, + where the nodes have _more_ than 2/3 of the entire committee's accumulated weight. +* Conceptually, Flow allows that the random beacon is run only by a subset of consensus nodes, aka the "random beacon committee". +* The messages from zero-weighted nodes are ignored by all committee members. -In addition to HotStuff's requirements on block validity, the Flow protocol adds additional requirements. + +### Determining block validity + +In addition to Jolteon's requirements on block validity, the Flow protocol adds additional requirements. For example, it is illegal to repeatedly include the same payload entities (e.g. collections, challenges, etc) in the same fork. -Generally, payload entities expire. However within the expiry horizon, all ancestors of a block need to be known +Generally, payload entities expire. However, within the expiry horizon, all ancestors of a block need to be known to verify that payload entities are not repeated. -We exclude the entire logic for determining payload validity from the HotStuff core implementation. +We exclude the entire logic for determining payload validity from the HotStuff core implementation. This functionality is encapsulated in the Chain Compliance Layer (CCL) which precedes HotStuff. +The CCL is designed to forward only fully validated blocks to the HotStuff core logic. The CCL forwards a block to the HotStuff core logic only if +* the block's header is valid (including QC and optional TC), * the block's payload is valid, * the block is connected to the most recently finalized block, and -* all ancestors have previously been processed by HotStuff. +* all ancestors have previously been forwarded to HotStuff. If ancestors of a block are missing, the CCL caches the respective block and (iteratively) requests missing ancestors. -#### Payload generation -Payloads are generated outside of the HotStuff core logic. HotStuff only incorporates the payload root hash into the block header. +### Payload generation +Payloads are generated outside the HotStuff core logic. HotStuff only incorporates the payload root hash into the block header. -#### Structure of votes +### Structure of votes In Flow's HotStuff implementation, votes are used for two purposes: -1. Prove that a super-majority of committee nodes considers the respective block a valid extension of the chain. +1. Prove that a super-majority of committee nodes consider the respective block a valid extension of the chain. Therefore, nodes include a `StakingSignature` (BLS with curve BLS12-381) in their vote. 2. Construct a Source of Randomness as described in [Dfinity's Random Beacon](https://dfinity.org/pdf-viewer/library/dfinity-consensus.pdf). Therefore, consensus nodes include a `RandomBeaconSignature` (also BLS with curve BLS12-381, used in a threshold signature scheme) in their vote. -When the primary collects the votes, it verifies the `StakingSignature` and the `RandomBeaconSignature`. +When the primary collects the votes, it verifies the content of `SigData`, which can contain only a `StakingSignature` or a pair `StakingSignature` + `RandomBeaconSignature`. +A `StakingSignature` must be present in all votes. (There is an optimization already implemented in the code, making the `StakingSignature` optional, but it is not enabled.) If either signature is invalid, the entire vote is discarded. From all valid votes, the -`StakingSignatures` and the `RandomBeaconSignatures` are aggregated separately. -(note: Only the aggregation of the `RandomBeaconSignature` into a threshold signature is currently implemented. -Aggregation of the BLS `StakingSignatures` will be added later.) +`StakingSignatures` and the `RandomBeaconSignatures` are aggregated separately. + +For purely consensus-theoretical purposes, it would be sufficient to use a threshold signature scheme. However, thresholds signatures have the following two +important limitations, for which reason Flow uses aggregated signatures in addition: +* The threshold signature carries no information about who signed. Meaning with the threshold signature alone, we have to way to distinguish + the nodes are contributing from the ones being offline. The mature flow protocol will reward nodes based on their contributions to QCs, + which requires a conventional aggregated signature. +* Furthermore, the distributed key generation [DKG] for threshold keys currently limits the number of nodes. By including a signature aggregate, + we can scale the consensus committee somewhat beyond the limitations of the [DKG]. Then, the nodes contributing to the random beacon would only + be a subset of the entire consensus committee. + +### Communication topology * Following [version 6 of the HotStuff paper](https://arxiv.org/abs/1803.05069v6), replicas forward their votes for block `b` to the leader of the _next_ view, i.e. the primary for view `b.View + 1`. @@ -68,69 +140,90 @@ replicas forward their votes for block `b` to the leader of the _next_ view, i.e (instead of signing the block proposal for authenticity and separately sending a vote). -#### Primary section +### Primary section For primary section, we use a randomized, weight-proportional selection. -### Component Overview +## Implementation Components HotStuff's core logic is broken down into multiple components. The figure below illustrates the dependencies of the core components and information flow between these components. -![](/docs/HotStuffEventLoop.png) - -* `HotStuffEngine` and `ChainComplianceLayer` do not contain any core HotStuff logic. -They are responsible for relaying HotStuff messages and validating block payloads respectively. -* `EventLoop` buffers all incoming events, so `EventHandler` can process one event at a time in a single thread. -* `EventHandler` orchestrates all HotStuff components and implements [HotStuff's state machine](/docs/StateMachine.png). -The event handler is designed to be executed single-threaded. -* `Communicator` relays outgoing HotStuff messages (votes and block proposals) -* `Pacemaker` is a basic PaceMaker to ensure liveness by keeping the majority of committee replicas in the same view +![](/docs/ComponentInteraction.png) +<!--- source: https://drive.google.com/file/d/1rZsYta0F9Uz5_HM84MlMmMbiR62YALX-/ --> + +* `MessageHub` is responsible for relaying HotStuff messages. Incoming messages are relayed to the respective modules depending on their message type. +Outgoing messages are relayed to the committee though the networking layer via epidemic gossip ('broadcast') or one-to-one communication ('unicast'). +* `compliance.Engine` is responsible for processing incoming blocks, caching if needed, validating, extending state and forwarding them to HotStuff for further processing. + Note: The embedded `compliance.Core` component is responsible for business logic and maintaining state; `compliance.Engine` schedules work and manages worker threads for the `Core`. +* `EventLoop` buffers all incoming events. It manages a single worker routine executing the EventHandler`'s logic. +* `EventHandler` orchestrates all HotStuff components and implements the [HotStuff's state machine](/docs/StateMachine.png). +The event handler is designed to be executed single-threaded. +* `SafetyRules` tracks the latest vote, the latest timeout and determines whether to vote for a block and if it's safe to timeout current round. +* `Pacemaker` implements Jolteon's PaceMaker. It manages and updates a replica's local view and synchronizes it with other replicas. + The `Pacemaker` ensures liveness by keeping a supermajority of the committee in the same view. * `Forks` maintains an in-memory representation of all blocks `b`, whose view is larger or equal to the view of the latest finalized block (known to this specific replica). -As blocks with missing ancestors are cached outside of HotStuff (by the Chain Compliance Layer), -all blocks stored in `Forks` are guaranteed to be connected to the genesis block -(or the trusted checkpoint from which the replica started). Internally, `Forks` consists of multiple layers: - - `LevelledForest`: A blockchain forms a Tree. When removing all blocks with views strictly smaller than the last finalized block, the chain decomposes into multiple disconnected trees. In graph theory, such structure is a forest. To separate general graph-theoretical concepts from the concrete block-chain application, `LevelledForest` refers to blocks as graph `vertices` and to a block's view number as `level`. The `LevelledForest` is an in-memory data structure to store and maintain a levelled forest. It provides functions to add vertices, query vertices by their ID (block's hash), query vertices by level, query the children of a vertex, and prune vertices by level (remove them from memory). - - `Finalizer`: the finalizer uses the `LevelledForest` to store blocks. The Finalizer tracks the locked and finalized blocks. Furthermore, it evaluates whether a block is safe to vote for. - - `ForkChoice`: implements the fork choice rule. Currently, `NewestForkChoice` always uses the quorum certificate (QC) with the largest view to build a new block (i.e. the fork a super-majority voted for most recently). - - `Forks` is the highest level. It bundles the `Finalizer` and `ForkChoice` into one component. + As blocks with missing ancestors are cached outside HotStuff (by the Chain Compliance Layer), + all blocks stored in `Forks` are guaranteed to be connected to the genesis block + (or the trusted checkpoint from which the replica started). `Forks` tracks the finalized blocks and triggers finalization events whenever it + observes a valid extension to the chain of finalized blocks. `Forks` is implemented using `LevelledForest`: + - Conceptually, a blockchain constructs a tree of blocks. When removing all blocks + with views strictly smaller than the last finalized block, this graph decomposes into multiple disconnected trees (referred to as a forest in graph theory). + `LevelledForest` is an in-memory data structure to store and maintain a levelled forest. It provides functions to add vertices, query vertices by their ID + (block's hash), query vertices by level (block's view), query the children of a vertex, and prune vertices by level (remove them from memory). + To separate general graph-theoretical concepts from the concrete blockchain application, `LevelledForest` refers to blocks as graph `vertices` + and to a block's view number as `level`. * `Validator` validates the HotStuff-relevant aspects of - - QC: total weight of all signers is more than 2/3 of committee weight, validity of signatures, view number is strictly monotonously increasing - - block proposal: from designated primary for the block's respective view, contains proposer's vote for its own block, QC in block is valid - - vote: validity of signature, voter is has positive weight + - QC: total weight of all signers is more than 2/3 of committee weight, validity of signatures, view number is strictly monotonously increasing; + - TC: total weight of all signers is more than 2/3 of committee weight, validity of signatures, proof for entering view; + - block proposal: from designated primary for the block's respective view, contains proposer's vote for its own block, QC in block is valid, + a valid TC for the previous view is included if and only if the QC is not for the previous view; + - vote: validity of signature, voter is has positive weight. * `VoteAggregator` caches votes on a per-block basis and builds QC if enough votes have been accumulated. -* `Voter` tracks the view of the latest vote and determines whether or not to vote for a block (by calling `forks.IsSafeBlock`) -* `Committee` maintains the list of all authorized network members and their respective weight on a per-block basis. Furthermore, the committee contains the primary selection algorithm. +* `TimeoutAggregator` caches timeouts on a per-view basis and builds TC if enough timeouts have been accumulated. Performs validation and verification of timeouts. +* `Replicas` maintains the list of all authorized network members and their respective weight, queryable by view. + It maintains a static list, which changes only between epochs. Furthermore, `Replicas` knows the primary for each view. +* `DynamicCommittee` maintains the list of all authorized network members and their respective weight on a per-block basis. + It extends `Replicas` allowing for committee changes mid epoch, e.g. due to slashing or node ejection. * `BlockProducer` constructs the payload of a block, after the HotStuff core logic has decided which fork to extend # Implementation -We have translated the Chained HotStuff protocol into a state machine shown below. The state machine is implemented -in `EventHandler`. +We have translated the HotStuff protocol into the state machine shown below. The state machine is implemented in `EventHandler`. ![](/docs/StateMachine.png) +<!--- source: https://drive.google.com/file/d/1la4jxyaEJJfip7NCWBM9YBTz6PK4-N9e/ --> + #### PaceMaker -The HotStuff state machine interacts with the PaceMaker, which triggers view changes. Conceptually, the PaceMaker -interfaces with the `EventHandler` in two different modes: +The HotStuff state machine interacts with the PaceMaker, which triggers view changes. The PaceMaker keeps track of +liveness data (newest QC, current view, TC for last view), and updates it when supplied with new data from `EventHandler`. +Conceptually, the PaceMaker interfaces with the `EventHandler` in two different modes: * [asynchronous] On timeouts, the PaceMaker will emit a timeout event, which is processed as any other event (such as incoming blocks or votes) through the `EventLoop`. -* [synchronous] When progress is made following the happy-path business logic, the `EventHandler` will inform the PaceMaker about completing the respective -processing step via a direct method call (see `PaceMaker interface`). If the PaceMaker changed the view in response, it returns +* [synchronous] When progress is made following the core business logic, the `EventHandler` will inform the PaceMaker about discovering new QCs or TCs +via a direct method call (see `PaceMaker interface`). If the PaceMaker changed the view in response, it returns a `NewViewEvent` which will be synchronously processed by the `EventHandler`. -Flow's PaceMaker is simple (compared to Libra's PaceMaker for instance). It solely relies on increasing the timeouts -exponentially if no progress is made and exponentially decreasing timeouts on the happy path. +Flow's PaceMaker utilizes dedicated messages for synchronizing the consensus participant's local views. +It broadcasts a `TimeoutObject`, whenever no progress is made during the current round. After collecting timeouts from a supermajority of participants, +the replica constructs a TC which can be used to enter the next round `V = TC.View + 1`. For calculating round timeouts we use a truncated exponential backoff. +We will increase round duration exponentially if no progress is made and exponentially decrease timeouts on happy path. +During normal operation with some benign crash failures, a small number of `k` subsequent leader failures is expected. +Therefore, our PaceMaker tolerates a few failures (`k=6`) before starting to increase timeouts, which is valuable for quickly +skipping over the offline replicase. However, the probability of `k` subsequent leader failures decreases exponentially with `k` (due to Flow's randomized leader selection). +Therefore, beyond `k=6`, we start increasing timeouts. The timeout values are limited by lower and upper-bounds to ensure that the PaceMaker can change from large to small timeouts in a reasonable number of views. -The specific values for lower and upper timeout bounds are protocol-specified; we envision the bounds to be on the order of 1sec (lower bound) and multiple days (upper bound). +The specific values for lower and upper timeout bounds are protocol-specified; we envision the bounds to be on the order of 1sec (lower bound) and one minute (upper bound). -**Progress**, from the perspective of the PaceMaker is defined as entering view `V` -for which the replica knows a QC with `V = QC.view + 1`. -In other words, we transition into the next view due to reaching quorum in the last view. +**Progress**, from the perspective of the PaceMaker, is defined as entering view `V` +for which the replica knows a QC or a TC with `V = QC.view + 1` or `V = TC.view + 1`. +In other words, we transition into the next view when observing a quorum from the last view. +In contrast to HotStuff, Jolteon only allows a transition into view `V+1` after observing a valid quorum for view `V`. There is no other, passive method for honest nodes to change views. A central, non-trivial functionality of the PaceMaker is to _skip views_. -Specifically, given a QC with view `qc.view`, the Pacemaker will skip ahead to view `qc.view + 1` if `currentView ≤ qc.view`. - -<img src="https://github.com/onflow/flow-go/blob/master/docs/PaceMaker.png" width="200"> - +Specifically, given a QC or TC with view `V`, the Pacemaker will skip ahead to view `V + 1` if `currentView ≤ V`. + +![](/docs/PaceMaker.png) + <!--- source: https://drive.google.com/file/d/1la4jxyaEJJfip7NCWBM9YBTz6PK4-N9e/ --> ## Code structure @@ -141,49 +234,50 @@ When starting to look into the code, we suggest starting with `/consensus/hotstu ### Folder structure -All files in the `/consensus/hotstuff/` folder, except for `event_loop.go` and `follower_loop.go`, are interfaces for HotStuff-related components. +All files in the `/consensus/hotstuff/` folder, except for `follower_loop.go`, are interfaces for HotStuff-related components. The concrete implementations for all HotStuff-relevant components are in corresponding sub-folders. -For completeness, we list the component implemented in each subfolder below: +For completeness, we list the component implemented in each sub-folder below: * `/consensus/hotstuff/blockproducer` builds a block proposal for a specified QC, interfaces with the logic for assembling a block payload, combines all relevant fields into a new block proposal. -* `/consensus/hotstuff/committee` maintains the list of all authorized network members and their respective weight on a per-block basis; contains the primary selection algorithm. +* `/consensus/hotstuff/committees` maintains the list of all authorized network members and their respective weight on a per-block and per-view basis depending on implementation; contains the primary selection algorithm. +* `/consensus/hotstuff/eventloop` buffers all incoming events, so `EventHandler` can process one event at a time in a single thread. * `/consensus/hotstuff/eventhandler` orchestrates all HotStuff components and implements the HotStuff state machine. The event handler is designed to be executed single-threaded. * `/consensus/hotstuff/follower` This component is only used by nodes that are _not_ participating in the HotStuff committee. As Flow has dedicated node roles with specialized network functions, only a subset of nodes run the full HotStuff protocol. Nevertheless, all nodes need to be able to act on blocks being finalized. The approach we have taken for Flow is that block proposals are broadcast to all nodes (including non-committee nodes). Non-committee nodes locally determine block finality by applying HotStuff's finality rules. The HotStuff Follower contains the functionality to consume block proposals and trigger downstream processing of finalized blocks. The Follower does not _actively_ participate in HotStuff. -* `/consensus/hotstuff/forks` maintains an in-memory representation of all blocks `b`, whose view is larger or equal to the view of the latest finalized block (known to this specific HotStuff replica). -It tracks the last finalized block, the currently locked block, evaluates whether it is safe to vote for a given block and provides a Fork-Choice when the replica is primary. +* `/consensus/hotstuff/forks` maintains an in-memory representation of blocks, whose view is larger or equal to the view of the latest finalized block (known to this specific HotStuff replica). Per convention, all blocks stored in `forks` passed validation and their ancestry is fully known. `forks` tracks the last finalized block and implements the 2-chain finalization rule. Specifically, we finalize block `B`, if a _certified_ child `B'` is known that was produced in the view `B.View +1`. * `/consensus/hotstuff/helper` contains broadly-used helper functions for testing * `/consensus/hotstuff/integration` integration tests for verifying correct interaction of multiple HotStuff replicas -* `/consensus/hotstuff/model` contains the HotStuff data models, including block proposal, QC, etc. -Many HotStuff data models are built on top of basic data models defined in `/model/flow/`. -* `/consensus/hotstuff/notifications`: All relevant events within the HotStuff logic are exported though a notification system. While the notifications are _not_ used HotStuff-internally, they notify other components within the same node of relevant progress and are used for collecting HotStuff metrics. -* `/consensus/hotstuff/pacemaker` contains the implementation of Flow's basic PaceMaker, as described above. -* `/consensus/hotstuff/persister` for performance reasons, the implementation maintains the consensus state largely in-memory. The `persister` stores the last entered view and the view of the latest voted block persistenlty on disk. This allows recovery after a crash without the risk of equivocation. -* `/consensus/hotstuff/runner` helper code for starting and shutting down the HotStuff logic safely in a multithreaded environment. -* `/consensus/hotstuff/validator` holds the logic for validating the HotStuff-relevant aspects of blocks, QCs, and votes +* `/consensus/hotstuff/model` contains the HotStuff data models, including block proposal, vote, timeout, etc. + Many HotStuff data models are built on top of basic data models defined in `/model/flow/`. +* `/consensus/hotstuff/notifications`: All relevant events within the HotStuff logic are exported though a notification system. Notifications are used by _some_ HotStuff components internally to drive core logic (e.g. events from `VoteAggregator` and `TimeoutAggregator` can trigger progress in the `EventHandler`). Furthermore, notifications inform other components within the same node of relevant progress and are used for collecting HotStuff metrics. Per convention, notifications are idempotent. +* `/consensus/hotstuff/pacemaker` contains the implementation of Flow's Active PaceMaker, as described above. Is responsible for protocol liveness. +* `/consensus/hotstuff/persister` stores the latest safety and liveness data _synchronously_ on disk. + The `persister` only covers the minimal amount of data that is absolutely necessary to avoid equivocation after a crash. + This data must be stored on disk immediately whenever updated, before the node can progress with its consensus logic. + In comparison, the majority of the consensus state is held in-memory for performance reasons and updated in an eventually consistent manner. + After a crash, some of this data might be lost (but can be re-requested) without risk of protocol violations. +* `/consensus/hotstuff/safetyrules` tracks the latest vote and the latest timeout. + It determines whether to vote for a block and if it's safe to construct a timeout for the current round. +* `/consensus/hotstuff/signature` contains the implementation for threshold signature aggregation for all types of signatures that are used in HotStuff protocol. +* `/consensus/hotstuff/timeoutcollector` encapsulates the logic for validating timeouts for one particular view and aggregating them to a TC. +* `/consensus/hotstuff/timeoutaggregator` orchestrates the `TimeoutCollector`s for different views. It distributes timeouts to the respective `TimeoutCollector` and prunes collectors that are no longer needed. +* `/consensus/hotstuff/tracker` implements utility code for tracking the newest QC and TC in a multithreaded environment. +* `/consensus/hotstuff/validator` holds the logic for validating the HotStuff-relevant aspects of blocks, QCs, TC, and votes * `/consensus/hotstuff/verification` contains integration of Flow's cryptographic primitives (signing and signature verification) -* `/consensus/hotstuff/voteaggregator` caches votes on a per-block basis and builds a QC if enough votes have been accumulated. -* `/consensus/hotstuff/voter` tracks the view of the latest vote and determines whether or not to vote for a block +* `/consensus/hotstuff/votecollector` encapsulates the logic for caching, validating, and aggregating votes for one particular view. + It tracks, whether a valid proposal for view is known and when enough votes have been collected, it builds a QC. +* `/consensus/hotstuff/voteaggregator` orchestrates the `VoteCollector`s for different views. It distributes votes to the respective `VoteCollector`, notifies the `VoteCollector` about the arrival of their respective block, and prunes collectors that are no longer needed. -## Pending Extensions -* BLS Aggregation of the `StakingSignatures` -* include Epochs -* upgrade PaceMaker to include Timeout Certificates -* refactor crypto integration (code in `verification` and dependent modules) for better auditability ## Telemetry The HotStuff state machine exposes some details about its internal progress as notification through the `hotstuff.Consumer`. The following figure depicts at which points notifications are emitted. -![](/docs/StateMachine_wirth_notifications.png) +![](/docs/StateMachine_with_notifications.png) + <!--- source: https://drive.google.com/file/d/1la4jxyaEJJfip7NCWBM9YBTz6PK4-N9e/ --> We have implemented a telemetry system (`hotstuff.notifications.TelemetryConsumer`) which implements the `Consumer` interface. -The `TelemetryConsumer` tracks all events as belonging together that were emitted during a path through the state machine. +The `TelemetryConsumer` tracks all events as belonging together that were emitted during a path through the state machine as well as events from components that perform asynchronous processing (`VoteAggregator`, `TimeoutAggregator`). Each `path` through the state machine is identified by a unique id. Generally, the `TelemetryConsumer` could export the collected data to a variety of backends. For now, we export the data to a logger. - - - - - diff --git a/consensus/hotstuff/block_producer.go b/consensus/hotstuff/block_producer.go index e3150d9edb5..0721380f51f 100644 --- a/consensus/hotstuff/block_producer.go +++ b/consensus/hotstuff/block_producer.go @@ -1,15 +1,15 @@ package hotstuff import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) -// BlockProducer builds a new block proposal by building a new block payload with the the builder module, +// BlockProducer builds a new block proposal by building a new block payload with the builder module, // and uses VoteCollectorFactory to create a disposable VoteCollector for producing the proposal vote. // BlockProducer assembles the new block proposal using the block payload, block header and the proposal vote. type BlockProducer interface { - // MakeBlockProposal builds a new HotStuff block proposal using the given view and - // the given quorum certificate for its parent. - MakeBlockProposal(qc *flow.QuorumCertificate, view uint64) (*model.Proposal, error) + // MakeBlockProposal builds a new HotStuff block proposal using the given view, + // the given quorum certificate for its parent and [optionally] a timeout certificate for last view(could be nil). + // No errors are expected during normal operation. + MakeBlockProposal(view uint64, qc *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*flow.Header, error) } diff --git a/consensus/hotstuff/blockproducer/block_producer.go b/consensus/hotstuff/blockproducer/block_producer.go index 8baafdab598..74d01cc317d 100644 --- a/consensus/hotstuff/blockproducer/block_producer.go +++ b/consensus/hotstuff/blockproducer/block_producer.go @@ -12,13 +12,16 @@ import ( // BlockProducer is responsible for producing new block proposals type BlockProducer struct { signer hotstuff.Signer - committee hotstuff.Committee + committee hotstuff.Replicas builder module.Builder } +var _ hotstuff.BlockProducer = (*BlockProducer)(nil) + // New creates a new BlockProducer which wraps the chain compliance layer block builder // to provide hotstuff with block proposals. -func New(signer hotstuff.Signer, committee hotstuff.Committee, builder module.Builder) (*BlockProducer, error) { +// No errors are expected during normal operation. +func New(signer hotstuff.Signer, committee hotstuff.Replicas, builder module.Builder) (*BlockProducer, error) { bp := &BlockProducer{ signer: signer, committee: committee, @@ -27,15 +30,19 @@ func New(signer hotstuff.Signer, committee hotstuff.Committee, builder module.Bu return bp, nil } -// MakeBlockProposal will build a proposal for the given view with the given QC -func (bp *BlockProducer) MakeBlockProposal(qc *flow.QuorumCertificate, view uint64) (*model.Proposal, error) { +// MakeBlockProposal builds a new HotStuff block proposal using the given view, +// the given quorum certificate for its parent and [optionally] a timeout certificate for last view(could be nil). +// No errors are expected during normal operation. +func (bp *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*flow.Header, error) { // the custom functions allows us to set some custom fields on the block; // in hotstuff, we use this for view number and signature-related fields setHotstuffFields := func(header *flow.Header) error { header.View = view + header.ParentView = qc.View header.ParentVoterIndices = qc.SignerIndices header.ParentVoterSigData = qc.SigData header.ProposerID = bp.committee.Self() + header.LastViewTC = lastViewTC // turn the header into a block header proposal as known by hotstuff block := model.Block{ @@ -63,8 +70,5 @@ func (bp *BlockProducer) MakeBlockProposal(qc *flow.QuorumCertificate, view uint return nil, fmt.Errorf("could not build block proposal on top of %v: %w", qc.BlockID, err) } - // turn the signed flow header into a proposal - proposal := model.ProposalFromFlow(header, qc.View) - - return proposal, nil + return header, nil } diff --git a/consensus/hotstuff/committee.go b/consensus/hotstuff/committee.go index be48e70798f..9b88ac769d1 100644 --- a/consensus/hotstuff/committee.go +++ b/consensus/hotstuff/committee.go @@ -5,78 +5,137 @@ import ( "github.com/onflow/flow-go/state/protocol" ) -// Committee accounts for the fact that we might have multiple HotStuff instances -// (collector committees and main consensus committee). Each HotStuff instance is supposed to -// have a dedicated Committee state. -// A Committee provides subset of the protocol.State, which is restricted to exactly those +// A committee provides a subset of the protocol.State, which is restricted to exactly those // nodes that participate in the current HotStuff instance: the state of all legitimate HotStuff -// participants for the specified block. Legitimate HotStuff participants have NON-ZERO WEIGHT. +// participants for the specified view. Legitimate HotStuff participants have NON-ZERO WEIGHT. // -// The intended use case is to support collectors running HotStuff within Flow. Specifically, -// the collectors produced their own blocks, independently of the Consensus Nodes (aka the main consensus). -// Given a collector block, some logic is required to find the main consensus block -// for determining the valid collector-HotStuff participants. -type Committee interface { - // Identities returns a IdentityList with legitimate HotStuff participants for the specified block. - // The returned list of HotStuff participants - // * contains nodes that are allowed to sign the specified block (legitimate participants with NON-ZERO WEIGHT) - // * is ordered in the canonical order - // * contains no duplicates. - // TODO: document possible error returns -> https://github.com/dapperlabs/flow-go/issues/6327 - Identities(blockID flow.Identifier) (flow.IdentityList, error) +// For the purposes of validating votes, timeouts, quorum certificates, and timeout certificates +// we consider a committee which is static over the course of an epoch. Although committee +// members may be ejected, or have their weight change during an epoch, we ignore these changes. +// For these purposes we use the Replicas and *ByEpoch methods. +// +// When validating proposals, we take into account changes to the committee during the course of +// an epoch. In particular, if a node is ejected, we will immediately reject all future proposals +// from that node. For these purposes we use the DynamicCommittee and *ByBlock methods. - // Identity returns the full Identity for specified HotStuff participant. - // The node must be a legitimate HotStuff participant with NON-ZERO WEIGHT at the specified block. - // ERROR conditions: - // * model.InvalidSignerError if participantID does NOT correspond to an authorized HotStuff participant at the specified block. - Identity(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) +// Replicas defines the consensus committee for the purposes of validating votes, timeouts, +// quorum certificates, and timeout certificates. Any consensus committee member who was +// authorized to contribute to consensus AT THE BEGINNING of the epoch may produce valid +// votes and timeouts for the entire epoch, even if they are later ejected. +// So for validating votes/timeouts we use *ByEpoch methods. +// +// Since the voter committee is considered static over an epoch: +// * we can query identities by view +// * we don't need the full block ancestry prior to validating messages +type Replicas interface { // LeaderForView returns the identity of the leader for a given view. // CAUTION: per liveness requirement of HotStuff, the leader must be fork-independent. // Therefore, a node retains its proposer view slots even if it is slashed. // Its proposal is simply considered invalid, as it is not from a legitimate participant. // Returns the following expected errors for invalid inputs: - // * epoch containing the requested view has not been set up (protocol.ErrNextEpochNotSetup) - // * epoch is too far in the past (leader.InvalidViewError) + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known LeaderForView(view uint64) (flow.Identifier, error) + // QuorumThresholdForView returns the minimum total weight for a supermajority + // at the given view. This weight threshold is computed using the total weight + // of the initial committee and is static over the course of an epoch. + // Returns the following expected errors for invalid inputs: + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known + QuorumThresholdForView(view uint64) (uint64, error) + + // TimeoutThresholdForView returns the minimum total weight of observed timeout objects + // required to safely timeout for the given view. This weight threshold is computed + // using the total weight of the initial committee and is static over the course of + // an epoch. + // Returns the following expected errors for invalid inputs: + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known + TimeoutThresholdForView(view uint64) (uint64, error) + // Self returns our own node identifier. // TODO: ultimately, the own identity of the node is necessary for signing. // Ideally, we would move the method for checking whether an Identifier refers to this node to the signer. // This would require some refactoring of EventHandler (postponed to later) Self() flow.Identifier - // DKG returns the DKG info for the given block. - DKG(blockID flow.Identifier) (DKG, error) + // DKG returns the DKG info for epoch given by the input view. + // Returns the following expected errors for invalid inputs: + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known + DKG(view uint64) (DKG, error) + + // IdentitiesByEpoch returns a list of the legitimate HotStuff participants for the epoch + // given by the input view. The list of participants is filtered by the provided selector. + // The returned list of HotStuff participants: + // * contains nodes that are allowed to submit votes or timeouts within the given epoch + // (un-ejected, non-zero weight at the beginning of the epoch) + // * is ordered in the canonical order + // * contains no duplicates. + // The list of all legitimate HotStuff participants for the given epoch can be obtained by using `filter.Any` + // + // CAUTION: DO NOT use this method for validating block proposals. + // + // Returns the following expected errors for invalid inputs: + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known + // + // TODO: should return identity skeleton https://github.com/dapperlabs/flow-go/issues/6232 + IdentitiesByEpoch(view uint64) (flow.IdentityList, error) + + // IdentityByEpoch returns the full Identity for specified HotStuff participant. + // The node must be a legitimate HotStuff participant with NON-ZERO WEIGHT at the specified block. + // ERROR conditions: + // * model.InvalidSignerError if participantID does NOT correspond to an authorized HotStuff participant at the specified block. + // + // Returns the following expected errors for invalid inputs: + // * model.ErrViewForUnknownEpoch if no epoch containing the given view is known + // + // TODO: should return identity skeleton https://github.com/dapperlabs/flow-go/issues/6232 + IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.Identity, error) +} + +// DynamicCommittee extends Replicas to provide the consensus committee for the purposes +// of validating proposals. The proposer committee reflects block-to-block changes in the +// identity table to support immediately rejecting proposals from nodes after they are ejected. +// For validating proposals, we use *ByBlock methods. +// +// Since the proposer committee can change at any block: +// * we query by block ID +// * we must have incorporated the full block ancestry prior to validating messages +type DynamicCommittee interface { + Replicas + + // IdentitiesByBlock returns a list of the legitimate HotStuff participants for the given block. + // The list of participants is filtered by the provided selector. + // The returned list of HotStuff participants: + // * contains nodes that are allowed to submit proposals, votes, and timeouts + // (un-ejected, non-zero weight at current block) + // * is ordered in the canonical order + // * contains no duplicates. + // The list of all legitimate HotStuff participants for the given epoch can be obtained by using `filter.Any` + // + // TODO - do we need this, if we are only checking a single proposer ID? + IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) + + // IdentityByBlock returns the full Identity for specified HotStuff participant. + // The node must be a legitimate HotStuff participant with NON-ZERO WEIGHT at the specified block. + // ERROR conditions: + // * model.InvalidSignerError if participantID does NOT correspond to an authorized HotStuff participant at the specified block. + IdentityByBlock(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) } -// BlockSignerDecoder defines how to convert the SignerIndices field within a particular -// block header to the identifiers of the nodes which signed the block. +// BlockSignerDecoder defines how to convert the ParentSignerIndices field within a +// particular block header to the identifiers of the nodes which signed the block. type BlockSignerDecoder interface { // DecodeSignerIDs decodes the signer indices from the given block header into full node IDs. + // Note: A block header contains a quorum certificate for its parent, which proves that the + // consensus committee has reached agreement on validity of parent block. Consequently, the + // returned IdentifierList contains the consensus participants that signed the parent block. // Expected Error returns during normal operations: - // * state.UnknownBlockError if block has not been ingested yet - // TODO: this sentinel could be changed to `ErrViewForUnknownEpoch` once we merge the active pacemaker - // * signature.InvalidSignerIndicesError if signer indices included in the header do - // not encode a valid subset of the consensus committee + // - model.ErrViewForUnknownEpoch if the given block's parent is within an unknown epoch + // - signature.InvalidSignerIndicesError if signer indices included in the header do + // not encode a valid subset of the consensus committee DecodeSignerIDs(header *flow.Header) (flow.IdentifierList, error) } type DKG interface { protocol.DKG } - -// ComputeWeightThresholdForBuildingQC returns the weight that is minimally required for building a QC -func ComputeWeightThresholdForBuildingQC(totalWeight uint64) uint64 { - // Given totalWeight, we need the smallest integer t such that 2 * totalWeight / 3 < t - // Formally, the minimally required weight is: 2 * Floor(totalWeight/3) + max(1, totalWeight mod 3) - floorOneThird := totalWeight / 3 // integer division, includes floor - res := 2 * floorOneThird - divRemainder := totalWeight % 3 - if divRemainder <= 1 { - res = res + 1 - } else { - res += divRemainder - } - return res -} diff --git a/consensus/hotstuff/committee_test.go b/consensus/hotstuff/committee_test.go deleted file mode 100644 index cf087a04ada..00000000000 --- a/consensus/hotstuff/committee_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package hotstuff_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/onflow/flow-go/consensus/hotstuff" -) - -// Test_ComputeWeightThreshold tests computing the HotStuff safety threshold. -func Test_ComputeWeightThreshold(t *testing.T) { - // testing lowest values - for i := 1; i <= 302; i++ { - threshold := hotstuff.ComputeWeightThresholdForBuildingQC(uint64(i)) - - boundaryValue := float64(i) * 2.0 / 3.0 - assert.True(t, boundaryValue < float64(threshold)) - assert.False(t, boundaryValue < float64(threshold-1)) - } -} diff --git a/consensus/hotstuff/committees/cluster_committee.go b/consensus/hotstuff/committees/cluster_committee.go index 99e52c8f73e..565261dd7ee 100644 --- a/consensus/hotstuff/committees/cluster_committee.go +++ b/consensus/hotstuff/committees/cluster_committee.go @@ -16,7 +16,7 @@ import ( // committees are epoch-scoped. // // Clusters build blocks on a cluster chain but must obtain identity table -// information from the main chain. Thus, block ID parameters in this Committee +// information from the main chain. Thus, block ID parameters in this DynamicCommittee // implementation reference blocks on the cluster chain, which in turn reference // blocks on the main chain - this implementation manages that translation. type Cluster struct { @@ -27,11 +27,15 @@ type Cluster struct { selection *leader.LeaderSelection // a filter that returns all members of the cluster committee allowed to vote clusterMemberFilter flow.IdentityFilter - // initial set of cluster members, WITHOUT updated weight + // initial set of cluster members, WITHOUT dynamic weight changes + // TODO: should use identity skeleton https://github.com/dapperlabs/flow-go/issues/6232 initialClusterMembers flow.IdentityList + weightThresholdForQC uint64 // computed based on initial cluster committee weights + weightThresholdForTO uint64 // computed based on initial cluster committee weights } -var _ hotstuff.Committee = (*Cluster)(nil) +var _ hotstuff.Replicas = (*Cluster)(nil) +var _ hotstuff.DynamicCommittee = (*Cluster)(nil) func NewClusterCommittee( state protocol.State, @@ -46,6 +50,7 @@ func NewClusterCommittee( return nil, fmt.Errorf("could not compute leader selection for cluster: %w", err) } + totalWeight := cluster.Members().TotalWeight() com := &Cluster{ state: state, payloads: payloads, @@ -57,13 +62,15 @@ func NewClusterCommittee( filter.HasWeight(true), ), initialClusterMembers: cluster.Members(), + weightThresholdForQC: WeightThresholdToBuildQC(totalWeight), + weightThresholdForTO: WeightThresholdToTimeout(totalWeight), } return com, nil } -// Identities returns the identities of all cluster members that are authorized to +// IdentitiesByBlock returns the identities of all cluster members that are authorized to // participate at the given block. The order of the identities is the canonical order. -func (c *Cluster) Identities(blockID flow.Identifier) (flow.IdentityList, error) { +func (c *Cluster) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) { // blockID is a collection block not a block produced by consensus, // to query the identities from protocol state, we need to use the reference block id from the payload // @@ -82,12 +89,11 @@ func (c *Cluster) Identities(blockID flow.Identifier) (flow.IdentityList, error) } // otherwise use the snapshot given by the reference block - identities, err := c.state.AtBlockID(payload.ReferenceBlockID).Identities(c.clusterMemberFilter) // remove ejected nodes - + identities, err := c.state.AtBlockID(payload.ReferenceBlockID).Identities(c.clusterMemberFilter) return identities, err } -func (c *Cluster) Identity(blockID flow.Identifier, nodeID flow.Identifier) (*flow.Identity, error) { +func (c *Cluster) IdentityByBlock(blockID flow.Identifier, nodeID flow.Identifier) (*flow.Identity, error) { // first retrieve the cluster block payload payload, err := c.payloads.ByBlockID(blockID) @@ -121,14 +127,56 @@ func (c *Cluster) Identity(blockID flow.Identifier, nodeID flow.Identifier) (*fl return identity, nil } +// IdentitiesByEpoch returns the initial cluster members for this epoch. The view +// parameter is the view in the cluster consensus. Since clusters only exist for +// one epoch, we don't need to check the view. +func (c *Cluster) IdentitiesByEpoch(_ uint64) (flow.IdentityList, error) { + return c.initialClusterMembers, nil +} + +// IdentityByEpoch returns the node from the initial cluster members for this epoch. +// The view parameter is the view in the cluster consensus. Since clusters only exist +// for one epoch, we don't need to check the view. +// +// Returns: +// - model.InvalidSignerError if nodeID was not listed by the Epoch Setup event as an +// authorized participant in this cluster +func (c *Cluster) IdentityByEpoch(_ uint64, nodeID flow.Identifier) (*flow.Identity, error) { + identity, ok := c.initialClusterMembers.ByNodeID(nodeID) + if !ok { + return nil, model.NewInvalidSignerErrorf("node %v is not an authorized hotstuff participant", nodeID) + } + return identity, nil +} + func (c *Cluster) LeaderForView(view uint64) (flow.Identifier, error) { return c.selection.LeaderForView(view) } +// QuorumThresholdForView returns the weight threshold required to build a QC +// for the given view. The view parameter is the view in the cluster consensus. +// Since clusters only exist for one epoch, and the weight threshold is static +// over the course of an epoch, we don't need to check the view. +// +// No errors are expected during normal operation. +func (c *Cluster) QuorumThresholdForView(_ uint64) (uint64, error) { + return c.weightThresholdForQC, nil +} + +// TimeoutThresholdForView returns the minimum weight of observed timeout objects to +// safely immediately timeout for the current view. The view parameter is the view +// in the cluster consensus. Since clusters only exist for one epoch, and the weight +// threshold is static over the course of an epoch, we don't need to check the view. +// +// No errors are expected during normal operation. +func (c *Cluster) TimeoutThresholdForView(_ uint64) (uint64, error) { + return c.weightThresholdForTO, nil +} + func (c *Cluster) Self() flow.Identifier { return c.me } -func (c *Cluster) DKG(_ flow.Identifier) (hotstuff.DKG, error) { +func (c *Cluster) DKG(_ uint64) (hotstuff.DKG, error) { panic("queried DKG of cluster committee") } diff --git a/consensus/hotstuff/committees/cluster_committee_test.go b/consensus/hotstuff/committees/cluster_committee_test.go index fbd6b6363f0..83903d23c3d 100644 --- a/consensus/hotstuff/committees/cluster_committee_test.go +++ b/consensus/hotstuff/committees/cluster_committee_test.go @@ -1,6 +1,7 @@ package committees import ( + "math/rand" "testing" "github.com/stretchr/testify/suite" @@ -68,6 +69,17 @@ func (suite *ClusterSuite) SetupTest() { suite.Require().NoError(err) } +// TestThresholds tests that the correct thresholds are returned. +func (suite *ClusterSuite) TestThresholds() { + threshold, err := suite.com.QuorumThresholdForView(rand.Uint64()) + suite.Require().NoError(err) + suite.Assert().Equal(WeightThresholdToBuildQC(suite.members.TotalWeight()), threshold) + + threshold, err = suite.com.TimeoutThresholdForView(rand.Uint64()) + suite.Require().NoError(err) + suite.Assert().Equal(WeightThresholdToTimeout(suite.members.TotalWeight()), threshold) +} + // TestInvalidSigner tests that the InvalidSignerError sentinel is // returned under the appropriate conditions. func (suite *ClusterSuite) TestInvalidSigner() { @@ -105,22 +117,30 @@ func (suite *ClusterSuite) TestInvalidSigner() { suite.Run("should return InvalidSignerError for non-existent signer", func() { suite.Run("root block", func() { - _, err := suite.com.Identity(rootBlockID, fakeID) + _, err := suite.com.IdentityByBlock(rootBlockID, fakeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) suite.Run("non-root block", func() { - _, err := suite.com.Identity(nonRootBlockID, fakeID) + _, err := suite.com.IdentityByBlock(nonRootBlockID, fakeID) + suite.Assert().True(model.IsInvalidSignerError(err)) + }) + suite.Run("by epoch", func() { + _, err := suite.com.IdentityByEpoch(rand.Uint64(), fakeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) }) suite.Run("should return InvalidSignerError for existent non-cluster-member", func() { suite.Run("root block", func() { - _, err := suite.com.Identity(rootBlockID, realNonClusterMember.NodeID) + _, err := suite.com.IdentityByBlock(rootBlockID, realNonClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) suite.Run("non-root block", func() { - _, err := suite.com.Identity(nonRootBlockID, realNonClusterMember.NodeID) + _, err := suite.com.IdentityByBlock(nonRootBlockID, realNonClusterMember.NodeID) + suite.Assert().True(model.IsInvalidSignerError(err)) + }) + suite.Run("by epoch", func() { + _, err := suite.com.IdentityByEpoch(rand.Uint64(), realNonClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) }) @@ -128,37 +148,52 @@ func (suite *ClusterSuite) TestInvalidSigner() { suite.Run("should return ErrInvalidSigner for existent but ejected cluster member", func() { // at the root block, the cluster member is not ejected yet suite.Run("root block", func() { - actual, err := suite.com.Identity(rootBlockID, realEjectedClusterMember.NodeID) + actual, err := suite.com.IdentityByBlock(rootBlockID, realEjectedClusterMember.NodeID) suite.Require().NoError(err) suite.Assert().Equal(realEjectedClusterMember, actual) }) suite.Run("non-root block", func() { - _, err := suite.com.Identity(nonRootBlockID, realEjectedClusterMember.NodeID) + _, err := suite.com.IdentityByBlock(nonRootBlockID, realEjectedClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) + suite.Run("by epoch", func() { + actual, err := suite.com.IdentityByEpoch(rand.Uint64(), realEjectedClusterMember.NodeID) + suite.Assert().NoError(err) + suite.Assert().Equal(realEjectedClusterMember, actual) + }) }) suite.Run("should return ErrInvalidSigner for existent but zero-weight cluster member", func() { // at the root block, the cluster member has its initial weight suite.Run("root block", func() { - actual, err := suite.com.Identity(rootBlockID, realNoWeightClusterMember.NodeID) + actual, err := suite.com.IdentityByBlock(rootBlockID, realNoWeightClusterMember.NodeID) suite.Require().NoError(err) suite.Assert().Equal(realNoWeightClusterMember, actual) }) suite.Run("non-root block", func() { - _, err := suite.com.Identity(nonRootBlockID, realNoWeightClusterMember.NodeID) + _, err := suite.com.IdentityByBlock(nonRootBlockID, realNoWeightClusterMember.NodeID) suite.Assert().True(model.IsInvalidSignerError(err)) }) + suite.Run("by epoch", func() { + actual, err := suite.com.IdentityByEpoch(rand.Uint64(), realNoWeightClusterMember.NodeID) + suite.Require().NoError(err) + suite.Assert().Equal(realNoWeightClusterMember, actual) + }) }) suite.Run("should return identity for existent cluster member", func() { suite.Run("root block", func() { - actual, err := suite.com.Identity(rootBlockID, realClusterMember.NodeID) + actual, err := suite.com.IdentityByBlock(rootBlockID, realClusterMember.NodeID) suite.Require().NoError(err) suite.Assert().Equal(realClusterMember, actual) }) suite.Run("non-root block", func() { - actual, err := suite.com.Identity(nonRootBlockID, realClusterMember.NodeID) + actual, err := suite.com.IdentityByBlock(nonRootBlockID, realClusterMember.NodeID) + suite.Require().NoError(err) + suite.Assert().Equal(realClusterMember, actual) + }) + suite.Run("by epoch", func() { + actual, err := suite.com.IdentityByEpoch(rand.Uint64(), realClusterMember.NodeID) suite.Require().NoError(err) suite.Assert().Equal(realClusterMember, actual) }) diff --git a/consensus/hotstuff/committees/consensus_committee.go b/consensus/hotstuff/committees/consensus_committee.go index bf754b33d3f..c61ff5941ac 100644 --- a/consensus/hotstuff/committees/consensus_committee.go +++ b/consensus/hotstuff/committees/consensus_committee.go @@ -1,68 +1,187 @@ package committees import ( - "errors" "fmt" "sync" + "go.uber.org/atomic" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees/leader" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/events" "github.com/onflow/flow-go/state/protocol/seed" ) -var errSelectionNotComputed = fmt.Errorf("leader selection for epoch not yet computed") +// staticEpochInfo contains leader selection and the initial committee for one epoch. +// This data structure must not be mutated after construction. +type staticEpochInfo struct { + firstView uint64 // first view of the epoch (inclusive) + finalView uint64 // final view of the epoch (inclusive) + randomSource []byte // random source of epoch + leaders *leader.LeaderSelection // pre-computed leader selection for the epoch + // TODO: should use identity skeleton https://github.com/dapperlabs/flow-go/issues/6232 + initialCommittee flow.IdentityList + initialCommitteeMap map[flow.Identifier]*flow.Identity + weightThresholdForQC uint64 // computed based on initial committee weights + weightThresholdForTO uint64 // computed based on initial committee weights + dkg hotstuff.DKG +} + +// newStaticEpochInfo returns the static epoch information from the epoch. +// This can be cached and used for all by-view queries for this epoch. +func newStaticEpochInfo(epoch protocol.Epoch) (*staticEpochInfo, error) { + firstView, err := epoch.FirstView() + if err != nil { + return nil, fmt.Errorf("could not get first view: %w", err) + } + finalView, err := epoch.FinalView() + if err != nil { + return nil, fmt.Errorf("could not get final view: %w", err) + } + randomSource, err := epoch.RandomSource() + if err != nil { + return nil, fmt.Errorf("could not get random source: %w", err) + } + leaders, err := leader.SelectionForConsensus(epoch) + if err != nil { + return nil, fmt.Errorf("could not get leader selection: %w", err) + } + initialIdentities, err := epoch.InitialIdentities() + if err != nil { + return nil, fmt.Errorf("could not initial identities: %w", err) + } + initialCommittee := initialIdentities.Filter(filter.IsVotingConsensusCommitteeMember) + dkg, err := epoch.DKG() + if err != nil { + return nil, fmt.Errorf("could not get dkg: %w", err) + } + + totalWeight := initialCommittee.TotalWeight() + epochInfo := &staticEpochInfo{ + firstView: firstView, + finalView: finalView, + randomSource: randomSource, + leaders: leaders, + initialCommittee: initialCommittee, + initialCommitteeMap: initialCommittee.Lookup(), + weightThresholdForQC: WeightThresholdToBuildQC(totalWeight), + weightThresholdForTO: WeightThresholdToTimeout(totalWeight), + dkg: dkg, + } + return epochInfo, nil +} + +// newEmergencyFallbackEpoch creates an artificial fallback epoch generated from +// the last committed epoch at the time epoch emergency fallback is triggered. +// The fallback epoch: +// * begins after the last committed epoch +// * lasts until the next spork (estimated 6 months) +// * has the same static committee as the last committed epoch +func newEmergencyFallbackEpoch(lastCommittedEpoch *staticEpochInfo) (*staticEpochInfo, error) { + + rng, err := seed.PRGFromRandomSource(lastCommittedEpoch.randomSource, seed.ProtocolConsensusLeaderSelection) + if err != nil { + return nil, fmt.Errorf("could not create rng from seed: %w", err) + } + leaders, err := leader.ComputeLeaderSelection(lastCommittedEpoch.finalView+1, rng, leader.EstimatedSixMonthOfViews, lastCommittedEpoch.initialCommittee) + if err != nil { + return nil, fmt.Errorf("could not compute leader selection for fallback epoch: %w", err) + } + epochInfo := &staticEpochInfo{ + firstView: lastCommittedEpoch.finalView + 1, + finalView: lastCommittedEpoch.finalView + leader.EstimatedSixMonthOfViews, + randomSource: lastCommittedEpoch.randomSource, + leaders: leaders, + initialCommittee: lastCommittedEpoch.initialCommittee, + initialCommitteeMap: lastCommittedEpoch.initialCommitteeMap, + weightThresholdForQC: lastCommittedEpoch.weightThresholdForQC, + weightThresholdForTO: lastCommittedEpoch.weightThresholdForTO, + dkg: lastCommittedEpoch.dkg, + } + return epochInfo, nil +} // Consensus represents the main committee for consensus nodes. The consensus -// committee persists across epochs. +// committee might be active for multiple successive epochs. type Consensus struct { - mu sync.RWMutex - state protocol.State // the protocol state - me flow.Identifier // the node ID of this node - leaders map[uint64]*leader.LeaderSelection // pre-computed leader selection for each epoch + state protocol.State // the protocol state + me flow.Identifier // the node ID of this node + mu sync.RWMutex // protects access to epochs + epochs map[uint64]*staticEpochInfo // cache of initial committee & leader selection per epoch + committedEpochsCh chan *flow.Header // protocol events for newly committed epochs (the first block of the epoch is passed over the channel) + epochEmergencyFallback chan struct{} // protocol event for epoch emergency fallback + isEpochFallbackHandled *atomic.Bool // ensure we only inject fallback epoch once + events.Noop // implements protocol.Consumer + component.Component } -var _ hotstuff.Committee = (*Consensus)(nil) +var _ protocol.Consumer = (*Consensus)(nil) +var _ hotstuff.Replicas = (*Consensus)(nil) +var _ hotstuff.DynamicCommittee = (*Consensus)(nil) func NewConsensusCommittee(state protocol.State, me flow.Identifier) (*Consensus, error) { com := &Consensus{ - state: state, - me: me, - leaders: make(map[uint64]*leader.LeaderSelection), + state: state, + me: me, + epochs: make(map[uint64]*staticEpochInfo), + committedEpochsCh: make(chan *flow.Header, 1), + epochEmergencyFallback: make(chan struct{}, 1), + isEpochFallbackHandled: atomic.NewBool(false), } + com.Component = component.NewComponentManagerBuilder(). + AddWorker(com.handleProtocolEvents). + Build() + final := state.Final() - // pre-compute leader selection for current epoch - current := final.Epochs().Current() - _, err := com.prepareLeaderSelection(current) + // pre-compute leader selection for all presently relevant committed epochs + epochs := make([]protocol.Epoch, 0, 3) + // we always prepare the current epoch + epochs = append(epochs, final.Epochs().Current()) + + // we prepare the previous epoch, if one exists + exists, err := protocol.PreviousEpochExists(final) if err != nil { - return nil, fmt.Errorf("could not add leader for current epoch: %w", err) + return nil, fmt.Errorf("could not check previous epoch exists: %w", err) } - - // Pre-compute leader selection for previous epoch, if it exists. - // - // This ensures we always know about leader selection for at least one full - // epoch into the past, ensuring we are able to not only determine the leader - // for block proposals we receive, but also adjudicate consensus-related - // challenges up to one epoch into the past. - previous := final.Epochs().Previous() - _, err = previous.Counter() - // if there is no previous epoch, return the committee as-is - if errors.Is(err, protocol.ErrNoPreviousEpoch) { - return com, nil + if exists { + epochs = append(epochs, final.Epochs().Previous()) } + + // we prepare the next epoch, if it is committed + phase, err := final.Phase() if err != nil { - return nil, fmt.Errorf("could not get previous epoch: %w", err) + return nil, fmt.Errorf("could not check epoch phase: %w", err) + } + if phase == flow.EpochPhaseCommitted { + epochs = append(epochs, final.Epochs().Next()) } - _, err = com.prepareLeaderSelection(previous) + for _, epoch := range epochs { + _, err = com.prepareEpoch(epoch) + if err != nil { + return nil, fmt.Errorf("could not prepare initial epochs: %w", err) + } + } + + // if epoch emergency fallback was triggered, inject the fallback epoch + triggered, err := state.Params().EpochFallbackTriggered() if err != nil { - return nil, fmt.Errorf("could not add leader for previous epoch: %w", err) + return nil, fmt.Errorf("could not check epoch fallback: %w", err) + } + if triggered { + err = com.onEpochEmergencyFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not prepare emergency fallback epoch: %w", err) + } } return com, nil @@ -70,12 +189,12 @@ func NewConsensusCommittee(state protocol.State, me flow.Identifier) (*Consensus // Identities returns the identities of all authorized consensus participants at the given block. // The order of the identities is the canonical order. -func (c *Consensus) Identities(blockID flow.Identifier) (flow.IdentityList, error) { +func (c *Consensus) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) { il, err := c.state.AtBlockID(blockID).Identities(filter.IsVotingConsensusCommitteeMember) return il, err } -func (c *Consensus) Identity(blockID flow.Identifier, nodeID flow.Identifier) (*flow.Identity, error) { +func (c *Consensus) IdentityByBlock(blockID flow.Identifier, nodeID flow.Identifier) (*flow.Identity, error) { identity, err := c.state.AtBlockID(blockID).Identity(nodeID) if err != nil { if protocol.IsIdentityNotFound(err) { @@ -89,202 +208,285 @@ func (c *Consensus) Identity(blockID flow.Identifier, nodeID flow.Identifier) (* return identity, nil } -// LeaderForView returns the node ID of the leader for the given view. Returns -// the following errors: -// - epoch containing the requested view has not been set up (protocol.ErrNextEpochNotSetup) -// - epoch is too far in the past (leader.InvalidViewError) -// - any other error indicates an unexpected internal error -func (c *Consensus) LeaderForView(view uint64) (flow.Identifier, error) { +// IdentitiesByEpoch returns the committee identities in the epoch which contains +// the given view. +// +// Error returns: +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known. +// This is an expected error and must be handled. +// - unspecific error in case of unexpected problems and bugs +func (c *Consensus) IdentitiesByEpoch(view uint64) (flow.IdentityList, error) { + epochInfo, err := c.staticEpochInfoByView(view) + if err != nil { + return nil, err + } + return epochInfo.initialCommittee, nil +} - // try to retrieve the leader from a pre-computed LeaderSelection - id, err := c.precomputedLeaderForView(view) - if err == nil { - return id, nil +// IdentityByEpoch returns the identity for the given node ID, in the epoch which +// contains the given view. +// +// Error returns: +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known. +// This is an expected error and must be handled. +// - model.InvalidSignerError if nodeID was not listed by the Epoch Setup event as an +// authorized consensus participants. +// - unspecific error in case of unexpected problems and bugs +func (c *Consensus) IdentityByEpoch(view uint64, nodeID flow.Identifier) (*flow.Identity, error) { + epochInfo, err := c.staticEpochInfoByView(view) + if err != nil { + return nil, err } - if !errors.Is(err, errSelectionNotComputed) { - return flow.ZeroID, err + identity, ok := epochInfo.initialCommitteeMap[nodeID] + if !ok { + return nil, model.NewInvalidSignerErrorf("id %v is not a valid node id", nodeID) } - // we only reach the following code, if we got a errSelectionNotComputed - - // STEP 2 - we haven't yet computed leader selection for an epoch containing - // the requested view. We compute leader selection for the current and previous - // epoch (w.r.t. the finalized head) at initialization then compute leader - // selection for the next epoch when we encounter any view for which we don't - // know the leader. The series of epochs we have computed leaders for is - // strictly consecutive, meaning we know the leader for all views V where: - // - // oldestEpoch.firstView <= V <= newestEpoch.finalView - // - // Thus, the requested view is either before oldestEpoch.firstView or after - // newestEpoch.finalView. - // - // CASE 1: V < oldestEpoch.firstView - // If the view is before the first view we've computed the leader for, this - // represents an invalid query because we only guarantee the protocol state - // will contain epoch information for the current, previous, and next epoch - - // such a query must be for a view within an epoch at least TWO epochs before - // the current epoch when we started up. This is considered an invalid query. - // - // CASE 2: V > newestEpoch.finalView - // If the view is after the last view we've computed the leader for, we - // assume the view is within the next epoch (w.r.t. the finalized head). - // This assumption is equivalent to assuming that we build at least one - // block in every epoch, which is anyway a requirement for valid epochs. - // - epochs := c.state.Final().Epochs() - next := epochs.Next() - - // TMP: EMERGENCY EPOCH CHAIN CONTINUATION [EECC] - // - // If we reach this code-path, it means we are about to propose or vote - // for the first block in the next epoch. If that epoch has not been - // committed or set up, rather than stopping consensus, this intervention - // will create a new fallback leader selection for the next epoch containing - // 6 months worth of views, so that consensus will have leaders specified - // for the duration of the current spork, without any epoch transitions. - // - _, err = next.DKG() // either of the following errors indicates that we have transitioned into EECC - if errors.Is(err, protocol.ErrEpochNotCommitted) || errors.Is(err, protocol.ErrNextEpochNotSetup) { - current := epochs.Current() - - currentCounter, err := current.Counter() - if err != nil { - return flow.ZeroID, fmt.Errorf("could not get next epoch currentCounter: %w", err) - } - identities, err := current.InitialIdentities() - if err != nil { - return flow.ZeroID, fmt.Errorf("could not get epoch initial identities: %w", err) - } - // Get the random source - // CAUTION: this is re-using the same leader selection random source from the now-ending epoch - randomSeed, err := current.RandomSource() - if err != nil { - return flow.ZeroID, fmt.Errorf("could not get epoch seed: %w", err) - } - currentFinalView, err := current.FinalView() - if err != nil { - return flow.ZeroID, fmt.Errorf("could not get epoch first view: %w", err) - } - - // we will inject a fallback leader selection in place of the next epoch - counter := currentCounter + 1 - // the fallback leader selection begins after the final view of the current epoch - firstView := currentFinalView + 1 + return identity, nil +} - // create random number generator from the seed and customizer - rng, err := seed.PRGFromRandomSource(randomSeed, seed.ProtocolConsensusLeaderSelection) - if err != nil { - return flow.ZeroID, fmt.Errorf("could not create rng from seed: %w", err) - } +// LeaderForView returns the node ID of the leader for the given view. +// +// Error returns: +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known. +// This is an expected error and must be handled. +// - unspecific error in case of unexpected problems and bugs +func (c *Consensus) LeaderForView(view uint64) (flow.Identifier, error) { - selection, err := leader.ComputeLeaderSelection( - firstView, - rng, - int(firstView+leader.EstimatedSixMonthOfViews), // the fallback epoch lasts until the next spork - identities.Filter(filter.IsVotingConsensusCommitteeMember), - ) - if err != nil { - return flow.ZeroID, fmt.Errorf("could not compute epoch fallback leader selection: %w", err) - } - c.mu.Lock() - c.leaders[counter] = selection - c.mu.Unlock() - return selection.LeaderForView(view) + epochInfo, err := c.staticEpochInfoByView(view) + if err != nil { + return flow.ZeroID, err + } + leaderID, err := epochInfo.leaders.LeaderForView(view) + if leader.IsInvalidViewError(err) { + // an invalid view error indicates that no leader was computed for this view + // this is a fatal internal error, because the view necessarily is within an + // epoch for which we have pre-computed leader selection + return flow.ZeroID, fmt.Errorf("unexpected inconsistency in epoch view spans for view %d: %v", view, err) } if err != nil { - return flow.ZeroID, fmt.Errorf("unexpected error in EECC logic while retrieving DKG data: %w", err) + return flow.ZeroID, err } + return leaderID, nil +} - // HAPPY PATH logic - selection, err := c.prepareLeaderSelection(next) +// QuorumThresholdForView returns the minimum weight required to build a valid +// QC in the given view. The weight threshold only changes at epoch boundaries +// and is computed based on the initial committee weights. +// +// Error returns: +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known. +// This is an expected error and must be handled. +// - unspecific error in case of unexpected problems and bugs +func (c *Consensus) QuorumThresholdForView(view uint64) (uint64, error) { + epochInfo, err := c.staticEpochInfoByView(view) if err != nil { - return flow.ZeroID, fmt.Errorf("could not compute leader selection for next epoch: %w", err) + return 0, err } - - return selection.LeaderForView(view) + return epochInfo.weightThresholdForQC, nil } func (c *Consensus) Self() flow.Identifier { return c.me } -func (c *Consensus) DKG(blockID flow.Identifier) (hotstuff.DKG, error) { - return c.state.AtBlockID(blockID).Epochs().Current().DKG() +// TimeoutThresholdForView returns the minimum weight of observed timeout objects +// to safely immediately timeout for the current view. The weight threshold only +// changes at epoch boundaries and is computed based on the initial committee weights. +func (c *Consensus) TimeoutThresholdForView(view uint64) (uint64, error) { + epochInfo, err := c.staticEpochInfoByView(view) + if err != nil { + return 0, err + } + return epochInfo.weightThresholdForTO, nil } -// precomputedLeaderForView retrieves the leader from the precomputed -// LeaderSelection in `c.leaders` +// DKG returns the DKG for epoch which includes the given view. +// // Error returns: -// - errSelectionNotComputed [sentinel error] if there is no Epoch for view stored in `c.leaders` +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known. +// This is an expected error and must be handled. // - unspecific error in case of unexpected problems and bugs -func (c *Consensus) precomputedLeaderForView(view uint64) (flow.Identifier, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - // STEP 1 - look for an epoch matching this view for which we have already - // pre-computed leader selection. Epochs last ~500k views, so we find the - // epoch here 99.99% of the time. Since epochs are long-lived, it is fine - // for this to be linear in the number of epochs we have observed. - for _, selection := range c.leaders { - - // try retrieving the leader - leaderID, err := selection.LeaderForView(view) - // if the view is out of range, try the next epoch - if leader.IsInvalidViewError(err) { - continue - } - if err != nil { - return flow.ZeroID, fmt.Errorf("could not get leader: %w", err) +func (c *Consensus) DKG(view uint64) (hotstuff.DKG, error) { + epochInfo, err := c.staticEpochInfoByView(view) + if err != nil { + return nil, err + } + return epochInfo.dkg, nil +} + +// handleProtocolEvents processes queued Epoch events `EpochCommittedPhaseStarted` +// and `EpochEmergencyFallbackTriggered`. This function permanently utilizes a worker +// routine until the `Component` terminates. +// When we observe a new epoch being committed, we compute +// the leader selection and cache static info for the epoch. When we observe +// epoch emergency fallback being triggered, we inject a fallback epoch. +func (c *Consensus) handleProtocolEvents(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + for { + select { + case <-ctx.Done(): + return + case block := <-c.committedEpochsCh: + epoch := c.state.AtBlockID(block.ID()).Epochs().Next() + _, err := c.prepareEpoch(epoch) + if err != nil { + ctx.Throw(err) + } + case <-c.epochEmergencyFallback: + err := c.onEpochEmergencyFallbackTriggered() + if err != nil { + ctx.Throw(err) + } } + } +} + +// EpochCommittedPhaseStarted informs the `committee.Consensus` that the block starting the Epoch Committed Phase has been finalized. +func (c *Consensus) EpochCommittedPhaseStarted(_ uint64, first *flow.Header) { + c.committedEpochsCh <- first +} + +// EpochEmergencyFallbackTriggered passes the protocol event to the worker thread. +func (c *Consensus) EpochEmergencyFallbackTriggered() { + c.epochEmergencyFallback <- struct{}{} +} + +// onEpochEmergencyFallbackTriggered handles the protocol event for emergency epoch +// fallback mode being triggered. When this occurs, we inject a fallback epoch +// to the committee which extends the current epoch. +// This method must also be called on initialization, if emergency fallback mode +// was triggered in the past. +// No errors are expected during normal operation. +func (c *Consensus) onEpochEmergencyFallbackTriggered() error { + + // we respond to epoch fallback being triggered at most once, therefore + // the core logic is protected by an atomic bool. + // although it is only valid for epoch fallback to be triggered once per spork, + // we must account for repeated delivery of protocol events. + if !c.isEpochFallbackHandled.CompareAndSwap(false, true) { + return nil + } - return leaderID, nil + currentEpochCounter, err := c.state.Final().Epochs().Current().Counter() + if err != nil { + return fmt.Errorf("could not get current epoch counter: %w", err) + } + + c.mu.RLock() + // sanity check: current epoch must be cached already + currentEpoch, ok := c.epochs[currentEpochCounter] + if !ok { + c.mu.RUnlock() + return fmt.Errorf("epoch fallback: could not find current epoch (counter=%d) info", currentEpochCounter) + } + // sanity check: next epoch must never be committed, therefore must not be cached + _, ok = c.epochs[currentEpochCounter+1] + c.mu.RUnlock() + if ok { + return fmt.Errorf("epoch fallback: next epoch (counter=%d) is cached contrary to expectation", currentEpochCounter+1) + } + + fallbackEpoch, err := newEmergencyFallbackEpoch(currentEpoch) + if err != nil { + return fmt.Errorf("could not construct fallback epoch: %w", err) } - return flow.ZeroID, errSelectionNotComputed + // cache the epoch info + c.mu.Lock() + c.epochs[currentEpochCounter+1] = fallbackEpoch + c.mu.Unlock() + + return nil } -// prepareLeaderSelection pre-computes and stores the leader selection for the -// given epoch. Computing leader selection for the same epoch multiple times -// is a no-op. +// staticEpochInfoByView retrieves the previously cached static epoch info for +// the epoch which includes the given view. If no epoch is known for the given +// view, we will attempt to cache the next epoch. // -// Returns the leader selection for the given epoch. -func (c *Consensus) prepareLeaderSelection(epoch protocol.Epoch) (*leader.LeaderSelection, error) { - c.mu.Lock() - defer c.mu.Unlock() +// Error returns: +// - model.ErrViewForUnknownEpoch if no committed epoch containing the given view is known +// - unspecific error in case of unexpected problems and bugs +func (c *Consensus) staticEpochInfoByView(view uint64) (*staticEpochInfo, error) { + + // look for an epoch matching this view for which we have already pre-computed + // leader selection. Epochs last ~500k views, so we find the epoch here 99.99% + // of the time. Since epochs are long-lived and we only cache the most recent 3, + // this linear map iteration is inexpensive. + c.mu.RLock() + for _, epoch := range c.epochs { + if epoch.firstView <= view && view <= epoch.finalView { + c.mu.RUnlock() + return epoch, nil + } + } + c.mu.RUnlock() + + return nil, model.ErrViewForUnknownEpoch +} + +// prepareEpoch pre-computes and stores the static epoch information for the +// given epoch, including leader selection. Calling prepareEpoch multiple times +// for the same epoch returns cached static epoch information. +// Input must be a committed epoch. +// No errors are expected during normal operation. +func (c *Consensus) prepareEpoch(epoch protocol.Epoch) (*staticEpochInfo, error) { counter, err := epoch.Counter() if err != nil { - return nil, fmt.Errorf("could not get counter for current epoch: %w", err) + return nil, fmt.Errorf("could not get counter for epoch to prepare: %w", err) } - // this is a no-op if we have already computed leaders for this epoch - selection, exists := c.leaders[counter] + + // this is a no-op if we have already computed static info for this epoch + c.mu.RLock() + epochInfo, exists := c.epochs[counter] + c.mu.RUnlock() if exists { - return selection, nil + return epochInfo, nil } - selection, err = leader.SelectionForConsensus(epoch) + epochInfo, err = newStaticEpochInfo(epoch) if err != nil { - return nil, fmt.Errorf("could not get leader selection for current epoch: %w", err) + return nil, fmt.Errorf("could not create static epoch info for epch %d: %w", counter, err) + } + + // sanity check: ensure new epoch has contiguous views with the prior epoch + c.mu.RLock() + prevEpochInfo, exists := c.epochs[counter-1] + c.mu.RUnlock() + if exists { + if epochInfo.firstView != prevEpochInfo.finalView+1 { + return nil, fmt.Errorf("non-contiguous view ranges between consecutive epochs (epoch_%d=[%d,%d], epoch_%d=[%d,%d])", + counter-1, prevEpochInfo.firstView, prevEpochInfo.finalView, + counter, epochInfo.firstView, epochInfo.finalView) + } } - c.leaders[counter] = selection + // cache the epoch info + c.mu.Lock() + defer c.mu.Unlock() + c.epochs[counter] = epochInfo // now prune any old epochs, if we have exceeded our maximum of 3 // if we have fewer than 3 epochs, this is a no-op + c.pruneEpochInfo() + return epochInfo, nil +} +// pruneEpochInfo removes any epochs older than the most recent 3. +// NOTE: Not safe for concurrent use - the caller must first acquire the lock. +func (c *Consensus) pruneEpochInfo() { // find the maximum counter, including the epoch we just computed max := uint64(0) - for counter := range c.leaders { + for counter := range c.epochs { if counter > max { max = counter } } // remove any epochs which aren't within the most recent 3 - for counter := range c.leaders { + for counter := range c.epochs { if counter+3 <= max { - delete(c.leaders, counter) + delete(c.epochs, counter) } } - - return selection, nil } diff --git a/consensus/hotstuff/committees/consensus_committee_test.go b/consensus/hotstuff/committees/consensus_committee_test.go index 33985503707..20f59093d8b 100644 --- a/consensus/hotstuff/committees/consensus_committee_test.go +++ b/consensus/hotstuff/committees/consensus_committee_test.go @@ -1,15 +1,21 @@ package committees import ( + "context" "errors" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/mapfunc" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" protocolmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/state/protocol/seed" @@ -17,10 +23,243 @@ import ( "github.com/onflow/flow-go/utils/unittest/mocks" ) -// TestConsensus_InvalidSigner tests that the appropriate sentinel error is -// returned by the hotstuff.Committee implementation for non-existent or -// non-committee identities. -func TestConsensus_InvalidSigner(t *testing.T) { +func TestConsensusCommittee(t *testing.T) { + suite.Run(t, new(ConsensusSuite)) +} + +type ConsensusSuite struct { + suite.Suite + + // mocks + state *protocolmock.State + snapshot *protocolmock.Snapshot + params *protocolmock.Params + epochs *mocks.EpochQuery + + // backend for mocked functions + phase flow.EpochPhase + epochFallbackTriggered bool + currentEpochCounter uint64 + myID flow.Identifier + + committee *Consensus + cancel context.CancelFunc +} + +func (suite *ConsensusSuite) SetupTest() { + suite.phase = flow.EpochPhaseStaking + suite.epochFallbackTriggered = false + suite.currentEpochCounter = 1 + suite.myID = unittest.IdentifierFixture() + + suite.state = new(protocolmock.State) + suite.snapshot = new(protocolmock.Snapshot) + suite.params = new(protocolmock.Params) + suite.epochs = mocks.NewEpochQuery(suite.T(), suite.currentEpochCounter) + + suite.state.On("Final").Return(suite.snapshot) + suite.state.On("Params").Return(suite.params) + suite.params.On("EpochFallbackTriggered").Return( + func() bool { return suite.epochFallbackTriggered }, + func() error { return nil }, + ) + suite.snapshot.On("Phase").Return( + func() flow.EpochPhase { return suite.phase }, + func() error { return nil }, + ) + suite.snapshot.On("Epochs").Return(suite.epochs) +} + +func (suite *ConsensusSuite) TearDownTest() { + if suite.cancel != nil { + suite.cancel() + } +} + +// CreateAndStartCommittee instantiates and starts the committee. +// Should be called only once per test, after initial epoch mocks are created. +// It spawns a goroutine to detect fatal errors from the committee's error channel. +func (suite *ConsensusSuite) CreateAndStartCommittee() { + committee, err := NewConsensusCommittee(suite.state, suite.myID) + require.NoError(suite.T(), err) + ctx, cancel, errCh := irrecoverable.WithSignallerAndCancel(context.Background()) + committee.Start(ctx) + go unittest.FailOnIrrecoverableError(suite.T(), ctx.Done(), errCh) + + suite.committee = committee + suite.cancel = cancel +} + +// CommitEpoch adds the epoch to the protocol state and mimics the protocol state +// behaviour when committing an epoch, by sending the protocol event to the committee. +func (suite *ConsensusSuite) CommitEpoch(epoch protocol.Epoch) { + firstBlockOfCommittedPhase := unittest.BlockHeaderFixture() + suite.state.On("AtBlockID", firstBlockOfCommittedPhase.ID()).Return(suite.snapshot) + suite.epochs.Add(epoch) + suite.committee.EpochCommittedPhaseStarted(1, firstBlockOfCommittedPhase) + + // get the first view, to test when the epoch has been processed + firstView, err := epoch.FirstView() + require.NoError(suite.T(), err) + + // wait for the protocol event to be processed (async) + assert.Eventually(suite.T(), func() bool { + _, err := suite.committee.IdentitiesByEpoch(firstView) + return err == nil + }, time.Second, time.Millisecond) +} + +// AssertStoredEpochCounterRange asserts that the cached epochs are for exactly +// the given contiguous, inclusive counter range. +// Eg. for the input (2,4), the committee must have epochs cached with counters 2,3,4 +func (suite *ConsensusSuite) AssertStoredEpochCounterRange(from, to uint64) { + set := make(map[uint64]struct{}) + for i := from; i <= to; i++ { + set[i] = struct{}{} + } + + suite.committee.mu.RLock() + defer suite.committee.mu.RUnlock() + for epoch := range suite.committee.epochs { + delete(set, epoch) + } + + if !assert.Len(suite.T(), set, 0) { + suite.T().Logf("%v should be empty, but isn't; expected epoch range [%d,%d]", set, from, to) + } +} + +// TestConstruction_CurrentEpoch tests construction with only a current epoch. +// Only the current epoch should be cached after construction. +func (suite *ConsensusSuite) TestConstruction_CurrentEpoch() { + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(curEpoch) + + suite.CreateAndStartCommittee() + suite.Assert().Len(suite.committee.epochs, 1) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter) +} + +// TestConstruction_PreviousEpoch tests construction with a previous epoch. +// Both current and previous epoch should be cached after construction. +func (suite *ConsensusSuite) TestConstruction_PreviousEpoch() { + prevEpoch := newMockEpoch(suite.currentEpochCounter-1, unittest.IdentityListFixture(10), 1, 100, unittest.SeedFixture(32), true) + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(prevEpoch) + suite.epochs.Add(curEpoch) + + suite.CreateAndStartCommittee() + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter-1, suite.currentEpochCounter) +} + +// TestConstruction_UncommittedNextEpoch tests construction with an uncommitted next epoch. +// Only the current epoch should be cached after construction. +func (suite *ConsensusSuite) TestConstruction_UncommittedNextEpoch() { + suite.phase = flow.EpochPhaseSetup + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, unittest.IdentityListFixture(10), 201, 300, unittest.SeedFixture(32), false) + suite.epochs.Add(curEpoch) + suite.epochs.Add(nextEpoch) + + suite.CreateAndStartCommittee() + suite.Assert().Len(suite.committee.epochs, 1) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter) +} + +// TestConstruction_CommittedNextEpoch tests construction with a committed next epoch. +// Both current and next epochs should be cached after construction. +func (suite *ConsensusSuite) TestConstruction_CommittedNextEpoch() { + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, unittest.IdentityListFixture(10), 201, 300, unittest.SeedFixture(32), true) + suite.epochs.Add(curEpoch) + suite.epochs.Add(nextEpoch) + suite.phase = flow.EpochPhaseCommitted + + suite.CreateAndStartCommittee() + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) +} + +// TestConstruction_EpochFallbackTriggered tests construction when EECC has been triggered. +// Both current and the injected fallback epoch should be cached after construction. +func (suite *ConsensusSuite) TestConstruction_EpochFallbackTriggered() { + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(curEpoch) + suite.epochFallbackTriggered = true + + suite.CreateAndStartCommittee() + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) +} + +// TestProtocolEvents_CommittedEpoch tests that protocol events notifying of a newly +// committed epoch are handled correctly. A committed epoch should be cached, and +// repeated events should be no-ops. +func (suite *ConsensusSuite) TestProtocolEvents_CommittedEpoch() { + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(curEpoch) + + suite.CreateAndStartCommittee() + + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, unittest.IdentityListFixture(10), 201, 300, unittest.SeedFixture(32), true) + + firstBlockOfCommittedPhase := unittest.BlockHeaderFixture() + suite.state.On("AtBlockID", firstBlockOfCommittedPhase.ID()).Return(suite.snapshot) + suite.epochs.Add(nextEpoch) + suite.committee.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + // wait for the protocol event to be processed (async) + assert.Eventually(suite.T(), func() bool { + _, err := suite.committee.IdentitiesByEpoch(unittest.Uint64InRange(201, 300)) + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) + + // should handle multiple deliveries of the protocol event + suite.committee.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + suite.committee.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + suite.committee.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) + +} + +// TestProtocolEvents_EpochFallback tests that protocol events notifying of epoch +// fallback are handled correctly. Epoch fallback triggering should result in a +// fallback epoch being injected, and repeated events should be no-ops. +func (suite *ConsensusSuite) TestProtocolEvents_EpochFallback() { + curEpoch := newMockEpoch(suite.currentEpochCounter, unittest.IdentityListFixture(10), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(curEpoch) + + suite.CreateAndStartCommittee() + + suite.committee.EpochEmergencyFallbackTriggered() + // wait for the protocol event to be processed (async) + assert.Eventually(suite.T(), func() bool { + _, err := suite.committee.IdentitiesByEpoch(unittest.Uint64InRange(201, 300)) + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) + + // should handle multiple deliveries of the protocol event + suite.committee.EpochEmergencyFallbackTriggered() + suite.committee.EpochEmergencyFallbackTriggered() + suite.committee.EpochEmergencyFallbackTriggered() + + suite.Assert().Len(suite.committee.epochs, 2) + suite.AssertStoredEpochCounterRange(suite.currentEpochCounter, suite.currentEpochCounter+1) +} + +// TestIdentitiesByBlock tests retrieving committee members by block. +// * should use up-to-block committee information +// * should exclude non-committee members +func (suite *ConsensusSuite) TestIdentitiesByBlock() { + t := suite.T() realIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(0)) @@ -29,157 +268,295 @@ func TestConsensus_InvalidSigner(t *testing.T) { fakeID := unittest.IdentifierFixture() blockID := unittest.IdentifierFixture() - state := new(protocolmock.State) - snapshot := new(protocolmock.Snapshot) - // create a mock epoch for leader selection setup in constructor - currEpoch := newMockEpoch( - 1, - unittest.IdentityListFixture(10), - 1, - 100, - unittest.SeedFixture(seed.RandomSourceLength), - ) - epochs := mocks.NewEpochQuery(t, 1, currEpoch) - snapshot.On("Epochs").Return(epochs) - - state.On("Final").Return(snapshot) - state.On("AtBlockID", blockID).Return(snapshot) + currEpoch := newMockEpoch(1, unittest.IdentityListFixture(10), 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + suite.epochs.Add(currEpoch) - snapshot.On("Identity", realIdentity.NodeID).Return(realIdentity, nil) - snapshot.On("Identity", zeroWeightConsensusIdentity.NodeID).Return(zeroWeightConsensusIdentity, nil) - snapshot.On("Identity", ejectedConsensusIdentity.NodeID).Return(ejectedConsensusIdentity, nil) - snapshot.On("Identity", validNonConsensusIdentity.NodeID).Return(validNonConsensusIdentity, nil) - snapshot.On("Identity", fakeID).Return(nil, protocol.IdentityNotFoundError{}) + suite.state.On("AtBlockID", blockID).Return(suite.snapshot) + suite.snapshot.On("Identity", realIdentity.NodeID).Return(realIdentity, nil) + suite.snapshot.On("Identity", zeroWeightConsensusIdentity.NodeID).Return(zeroWeightConsensusIdentity, nil) + suite.snapshot.On("Identity", ejectedConsensusIdentity.NodeID).Return(ejectedConsensusIdentity, nil) + suite.snapshot.On("Identity", validNonConsensusIdentity.NodeID).Return(validNonConsensusIdentity, nil) + suite.snapshot.On("Identity", fakeID).Return(nil, protocol.IdentityNotFoundError{}) - com, err := NewConsensusCommittee(state, unittest.IdentifierFixture()) - require.NoError(t, err) + suite.CreateAndStartCommittee() t.Run("non-existent identity should return InvalidSignerError", func(t *testing.T) { - _, err := com.Identity(blockID, fakeID) + _, err := suite.committee.IdentityByBlock(blockID, fakeID) require.True(t, model.IsInvalidSignerError(err)) }) t.Run("existent but non-committee-member identity should return InvalidSignerError", func(t *testing.T) { t.Run("zero-weight consensus node", func(t *testing.T) { - _, err := com.Identity(blockID, zeroWeightConsensusIdentity.NodeID) + _, err := suite.committee.IdentityByBlock(blockID, zeroWeightConsensusIdentity.NodeID) require.True(t, model.IsInvalidSignerError(err)) }) t.Run("ejected consensus node", func(t *testing.T) { - _, err := com.Identity(blockID, ejectedConsensusIdentity.NodeID) + _, err := suite.committee.IdentityByBlock(blockID, ejectedConsensusIdentity.NodeID) require.True(t, model.IsInvalidSignerError(err)) }) t.Run("otherwise valid non-consensus node", func(t *testing.T) { - _, err := com.Identity(blockID, validNonConsensusIdentity.NodeID) + _, err := suite.committee.IdentityByBlock(blockID, validNonConsensusIdentity.NodeID) require.True(t, model.IsInvalidSignerError(err)) }) }) t.Run("should be able to retrieve real identity", func(t *testing.T) { - actual, err := com.Identity(blockID, realIdentity.NodeID) + actual, err := suite.committee.IdentityByBlock(blockID, realIdentity.NodeID) require.NoError(t, err) require.Equal(t, realIdentity, actual) }) + t.Run("should propagate unexpected errors", func(t *testing.T) { + mockErr := errors.New("unexpected") + suite.snapshot.On("Identity", mock.Anything).Return(nil, mockErr) + _, err := suite.committee.IdentityByBlock(blockID, unittest.IdentifierFixture()) + assert.ErrorIs(t, err, mockErr) + }) +} + +// TestIdentitiesByEpoch tests that identities can be queried by epoch. +// * should use static epoch info (initial identities) +// * should exclude non-committee members +// * should correctly map views to epochs +// * should return ErrViewForUnknownEpoch sentinel for unknown epochs +func (suite *ConsensusSuite) TestIdentitiesByEpoch() { + t := suite.T() + + // epoch 1 identities with varying conditions which would disqualify them + // from committee participation + realIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) + zeroWeightConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithWeight(0)) + ejectedConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus), unittest.WithEjected(true)) + validNonConsensusIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch1Identities := flow.IdentityList{realIdentity, zeroWeightConsensusIdentity, ejectedConsensusIdentity, validNonConsensusIdentity} + + // a single consensus node for epoch 2: + epoch2Identity := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus)) + epoch2Identities := flow.IdentityList{epoch2Identity} + + // create a mock epoch for leader selection setup in constructor + epoch1 := newMockEpoch(suite.currentEpochCounter, epoch1Identities, 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + // initially epoch 2 is not committed + epoch2 := newMockEpoch(suite.currentEpochCounter+1, epoch2Identities, 101, 200, unittest.SeedFixture(seed.RandomSourceLength), true) + suite.epochs.Add(epoch1) + + suite.CreateAndStartCommittee() + + t.Run("only epoch 1 committed", func(t *testing.T) { + t.Run("non-existent identity should return InvalidSignerError", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), unittest.IdentifierFixture()) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("existent but non-committee-member identity should return InvalidSignerError", func(t *testing.T) { + t.Run("zero-weight consensus node", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), zeroWeightConsensusIdentity.NodeID) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("ejected consensus node", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), ejectedConsensusIdentity.NodeID) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("otherwise valid non-consensus node", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), validNonConsensusIdentity.NodeID) + require.True(t, model.IsInvalidSignerError(err)) + }) + }) + + t.Run("should be able to retrieve real identity", func(t *testing.T) { + actual, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), realIdentity.NodeID) + require.NoError(t, err) + require.Equal(t, realIdentity, actual) + }) + + t.Run("should return ErrViewForUnknownEpoch for view outside existing epoch", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(101, 1_000_000), epoch2Identity.NodeID) + require.Error(t, err) + require.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + }) + }) + + // commit epoch 2 + suite.CommitEpoch(epoch2) + + t.Run("epoch 1 and 2 committed", func(t *testing.T) { + t.Run("should be able to retrieve epoch 1 identity in epoch 1", func(t *testing.T) { + actual, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), realIdentity.NodeID) + require.NoError(t, err) + require.Equal(t, realIdentity, actual) + }) + + t.Run("should be unable to retrieve epoch 1 identity in epoch 2", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(101, 200), realIdentity.NodeID) + require.Error(t, err) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("should be unable to retrieve epoch 2 identity in epoch 1", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(1, 100), epoch2Identity.NodeID) + require.Error(t, err) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("should be able to retrieve epoch 2 identity in epoch 2", func(t *testing.T) { + actual, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(101, 200), epoch2Identity.NodeID) + require.NoError(t, err) + require.Equal(t, epoch2Identity, actual) + }) + + t.Run("should return ErrViewForUnknownEpoch for view outside existing epochs", func(t *testing.T) { + _, err := suite.committee.IdentityByEpoch(unittest.Uint64InRange(201, 1_000_000), epoch2Identity.NodeID) + require.Error(t, err) + require.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + }) + }) + } -// test that LeaderForView returns a valid leader for the previous and current -// epoch and that it returns the appropriate sentinel for the next epoch if it -// is not yet ready -func TestConsensus_LeaderForView(t *testing.T) { +// TestThresholds tests that the weight threshold methods return the +// correct thresholds for the previous and current epoch and that it returns the +// appropriate sentinel for the next epoch if it is not yet ready. +// +// There are 3 epochs in this test case, each with the same identities but different +// weights. +func (suite *ConsensusSuite) TestThresholds() { + t := suite.T() identities := unittest.IdentityListFixture(10) - me := identities[0].NodeID - // the counter for the current epoch - epochCounter := uint64(2) + prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities.Map(mapfunc.WithWeight(100)), 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + currEpoch := newMockEpoch(suite.currentEpochCounter, identities.Map(mapfunc.WithWeight(200)), 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(prevEpoch) + suite.epochs.Add(currEpoch) - // create mocks - state := new(protocolmock.State) - snapshot := new(protocolmock.Snapshot) + suite.CreateAndStartCommittee() - prevEpoch := newMockEpoch( - epochCounter-1, - identities, - 1, - 100, - unittest.SeedFixture(seed.RandomSourceLength), - ) - currEpoch := newMockEpoch( - epochCounter, - identities, - 101, - 200, - unittest.SeedFixture(32), - ) + t.Run("next epoch not ready", func(t *testing.T) { + t.Run("previous epoch", func(t *testing.T) { + threshold, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(1, 100)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToBuildQC(1000), threshold) + threshold, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(1, 100)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToTimeout(1000), threshold) + }) - state.On("Final").Return(snapshot) - epochs := mocks.NewEpochQuery(t, epochCounter, prevEpoch, currEpoch) - snapshot.On("Epochs").Return(epochs) + t.Run("current epoch", func(t *testing.T) { + threshold, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(101, 200)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToBuildQC(2000), threshold) + threshold, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(101, 200)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToTimeout(2000), threshold) + }) - committee, err := NewConsensusCommittee(state, me) - require.Nil(t, err) + t.Run("after current epoch - should return ErrViewForUnknownEpoch", func(t *testing.T) { + // get threshold for view in next epoch when it is not set up yet + _, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(201, 300)) + assert.Error(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + _, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(201, 300)) + assert.Error(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + }) + }) + + // now, add a valid next epoch + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities.Map(mapfunc.WithWeight(300)), 201, 300, unittest.SeedFixture(seed.RandomSourceLength), true) + suite.CommitEpoch(nextEpoch) + + t.Run("next epoch ready", func(t *testing.T) { + t.Run("previous epoch", func(t *testing.T) { + threshold, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(1, 100)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToBuildQC(1000), threshold) + threshold, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(1, 100)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToTimeout(1000), threshold) + }) + + t.Run("current epoch", func(t *testing.T) { + threshold, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(101, 200)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToBuildQC(2000), threshold) + threshold, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(101, 200)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToTimeout(2000), threshold) + }) + + t.Run("next epoch", func(t *testing.T) { + threshold, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(201, 300)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToBuildQC(3000), threshold) + threshold, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(201, 300)) + require.Nil(t, err) + assert.Equal(t, WeightThresholdToTimeout(3000), threshold) + }) + + t.Run("beyond known epochs", func(t *testing.T) { + // get threshold for view in next epoch when it is not set up yet + _, err := suite.committee.QuorumThresholdForView(unittest.Uint64InRange(301, 10_000)) + assert.Error(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + _, err = suite.committee.TimeoutThresholdForView(unittest.Uint64InRange(301, 10_000)) + assert.Error(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + }) + }) +} + +// TestLeaderForView tests that LeaderForView returns a valid leader +// for the previous and current epoch and that it returns the appropriate +// sentinel for the next epoch if it is not yet ready +func (suite *ConsensusSuite) TestLeaderForView() { + t := suite.T() + + identities := unittest.IdentityListFixture(10) + + prevEpoch := newMockEpoch(suite.currentEpochCounter-1, identities, 1, 100, unittest.SeedFixture(seed.RandomSourceLength), true) + currEpoch := newMockEpoch(suite.currentEpochCounter, identities, 101, 200, unittest.SeedFixture(32), true) + suite.epochs.Add(currEpoch) + suite.epochs.Add(prevEpoch) + + suite.CreateAndStartCommittee() t.Run("next epoch not ready", func(t *testing.T) { t.Run("previous epoch", func(t *testing.T) { // get leader for view in previous epoch - leaderID, err := committee.LeaderForView(50) - require.Nil(t, err) + leaderID, err := suite.committee.LeaderForView(unittest.Uint64InRange(1, 100)) + assert.NoError(t, err) _, exists := identities.ByNodeID(leaderID) assert.True(t, exists) }) t.Run("current epoch", func(t *testing.T) { // get leader for view in current epoch - leaderID, err := committee.LeaderForView(150) - require.Nil(t, err) + leaderID, err := suite.committee.LeaderForView(unittest.Uint64InRange(101, 200)) + assert.NoError(t, err) _, exists := identities.ByNodeID(leaderID) assert.True(t, exists) }) - t.Run("after current epoch", func(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_TODO, "disabled as the current implementation uses a temporary fallback measure in this case (triggers EECC), rather than returning an error") - // REASON FOR SKIPPING TEST: - // We have a temporary fallback to continue with the current consensus committee, if the - // setup for the next epoch failed (aka emergency epoch chain continuation -- EECC). - // This test covers with behaviour _without_ EECC and is therefore skipped. - // The behaviour _with EECC_ is covered by the following test: - // "after current epoch - with emergency epoch chain continuation" - // TODO: for the mature implementation, remove EECC, enable this test, and remove the following test - + t.Run("after current epoch - should return ErrViewForUnknownEpoch", func(t *testing.T) { // get leader for view in next epoch when it is not set up yet - _, err := committee.LeaderForView(250) + _, err := suite.committee.LeaderForView(unittest.Uint64InRange(201, 300)) assert.Error(t, err) - assert.True(t, errors.Is(err, protocol.ErrNextEpochNotSetup)) - }) - - t.Run("after current epoch - with emergency epoch chain continuation", func(t *testing.T) { - // This test covers the TEMPORARY emergency epoch chain continuation (EECC) fallback - // TODO: for the mature implementation, remove this test, - // enable the previous test "after current epoch" - - // get leader for view in next epoch when it is not set up yet - _, err := committee.LeaderForView(250) - // emergency epoch chain continuation should kick in and return a valid leader - assert.NoError(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) }) }) // now, add a valid next epoch - nextEpoch := newMockEpoch( - epochCounter+1, - identities, - 201, - 300, - unittest.SeedFixture(seed.RandomSourceLength), - ) - epochs.Add(nextEpoch) + nextEpoch := newMockEpoch(suite.currentEpochCounter+1, identities, 201, 300, unittest.SeedFixture(seed.RandomSourceLength), true) + suite.CommitEpoch(nextEpoch) t.Run("next epoch ready", func(t *testing.T) { t.Run("previous epoch", func(t *testing.T) { // get leader for view in previous epoch - leaderID, err := committee.LeaderForView(50) + leaderID, err := suite.committee.LeaderForView(unittest.Uint64InRange(1, 100)) require.Nil(t, err) _, exists := identities.ByNodeID(leaderID) assert.True(t, exists) @@ -187,7 +564,7 @@ func TestConsensus_LeaderForView(t *testing.T) { t.Run("current epoch", func(t *testing.T) { // get leader for view in current epoch - leaderID, err := committee.LeaderForView(150) + leaderID, err := suite.committee.LeaderForView(unittest.Uint64InRange(101, 200)) require.Nil(t, err) _, exists := identities.ByNodeID(leaderID) assert.True(t, exists) @@ -195,14 +572,21 @@ func TestConsensus_LeaderForView(t *testing.T) { t.Run("next epoch", func(t *testing.T) { // get leader for view in next epoch after it has been set up - leaderID, err := committee.LeaderForView(250) + leaderID, err := suite.committee.LeaderForView(unittest.Uint64InRange(201, 300)) require.Nil(t, err) _, exists := identities.ByNodeID(leaderID) assert.True(t, exists) }) + + t.Run("beyond known epochs", func(t *testing.T) { + _, err := suite.committee.LeaderForView(unittest.Uint64InRange(301, 1_000_000)) + assert.Error(t, err) + assert.True(t, errors.Is(err, model.ErrViewForUnknownEpoch)) + }) }) } +// TestRemoveOldEpochs tests that old epochs are pruned func TestRemoveOldEpochs(t *testing.T) { identities := unittest.IdentityListFixture(10) @@ -213,22 +597,35 @@ func TestRemoveOldEpochs(t *testing.T) { currentEpochCounter := firstEpochCounter epochFinalView := uint64(100) - epoch1 := newMockEpoch(currentEpochCounter, identities, 1, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength)) + epoch1 := newMockEpoch(currentEpochCounter, identities, 1, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength), true) // create mocks state := new(protocolmock.State) snapshot := new(protocolmock.Snapshot) - + params := new(protocolmock.Params) state.On("Final").Return(snapshot) + state.On("Params").Return(params) + params.On("EpochFallbackTriggered").Return(false, nil) + epochQuery := mocks.NewEpochQuery(t, currentEpochCounter, epoch1) snapshot.On("Epochs").Return(epochQuery) + currentEpochPhase := flow.EpochPhaseStaking + snapshot.On("Phase").Return( + func() flow.EpochPhase { return currentEpochPhase }, + func() error { return nil }, + ) - committee, err := NewConsensusCommittee(state, me) + com, err := NewConsensusCommittee(state, me) require.Nil(t, err) + ctx, cancel, errCh := irrecoverable.WithSignallerAndCancel(context.Background()) + com.Start(ctx) + go unittest.FailOnIrrecoverableError(t, ctx.Done(), errCh) + defer cancel() + // we should start with only current epoch (epoch 1) pre-computed // since there is no previous epoch - assert.Equal(t, 1, len(committee.leaders)) + assert.Equal(t, 1, len(com.epochs)) // test for 10 epochs for currentEpochCounter < 10 { @@ -237,11 +634,21 @@ func TestRemoveOldEpochs(t *testing.T) { firstView := epochFinalView + 1 epochFinalView = epochFinalView + 100 currentEpochCounter++ - nextEpoch := newMockEpoch(currentEpochCounter, identities, firstView, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength)) + nextEpoch := newMockEpoch(currentEpochCounter, identities, firstView, epochFinalView, unittest.SeedFixture(seed.RandomSourceLength), true) epochQuery.Add(nextEpoch) + currentEpochPhase = flow.EpochPhaseCommitted + firstBlockOfCommittedPhase := unittest.BlockHeaderFixture() + state.On("AtBlockID", firstBlockOfCommittedPhase.ID()).Return(snapshot) + com.EpochCommittedPhaseStarted(currentEpochCounter, firstBlockOfCommittedPhase) + // wait for the protocol event to be processed (async) + require.Eventually(t, func() bool { + _, err := com.IdentityByEpoch(unittest.Uint64InRange(firstView, epochFinalView), unittest.IdentifierFixture()) + return !errors.Is(err, model.ErrViewForUnknownEpoch) + }, time.Second, time.Millisecond) + // query a view from the new epoch - _, err = committee.LeaderForView(firstView) + _, err = com.LeaderForView(firstView) require.NoError(t, err) // transition to the next epoch epochQuery.Transition() @@ -249,9 +656,9 @@ func TestRemoveOldEpochs(t *testing.T) { t.Run(fmt.Sprintf("epoch %d", currentEpochCounter), func(t *testing.T) { // check we have the right number of epochs stored if currentEpochCounter <= 3 { - assert.Equal(t, int(currentEpochCounter), len(committee.leaders)) + assert.Equal(t, int(currentEpochCounter), len(com.epochs)) } else { - assert.Equal(t, 3, len(committee.leaders)) + assert.Equal(t, 3, len(com.epochs)) } // check we have the correct epochs stored @@ -260,28 +667,27 @@ func TestRemoveOldEpochs(t *testing.T) { if counter < firstEpochCounter { break } - _, exists := committee.leaders[counter] + _, exists := com.epochs[counter] assert.True(t, exists, "missing epoch with counter %d max counter is %d", counter, currentEpochCounter) } }) } } -func newMockEpoch( - counter uint64, - identities flow.IdentityList, - firstView uint64, - finalView uint64, - seed []byte, -) *protocolmock.Epoch { +// newMockEpoch returns a new mocked epoch with the given fields +func newMockEpoch(counter uint64, identities flow.IdentityList, firstView uint64, finalView uint64, seed []byte, committed bool) *protocolmock.Epoch { epoch := new(protocolmock.Epoch) epoch.On("Counter").Return(counter, nil) epoch.On("InitialIdentities").Return(identities, nil) epoch.On("FirstView").Return(firstView, nil) epoch.On("FinalView").Return(finalView, nil) - // return nil error to indicate the epoch is committed - epoch.On("DKG").Return(nil, nil) + if committed { + // return nil error to indicate the epoch is committed + epoch.On("DKG").Return(nil, nil) + } else { + epoch.On("DKG").Return(nil, protocol.ErrNextEpochNotCommitted) + } epoch.On("RandomSource").Return(seed, nil) return epoch diff --git a/consensus/hotstuff/committees/metrics_wrapper.go b/consensus/hotstuff/committees/metrics_wrapper.go index 3ca30fbe075..e1bdcbc059a 100644 --- a/consensus/hotstuff/committees/metrics_wrapper.go +++ b/consensus/hotstuff/committees/metrics_wrapper.go @@ -9,35 +9,50 @@ import ( "github.com/onflow/flow-go/module" ) -// CommitteeMetricsWrapper implements the hotstuff.Committee interface. -// It wraps a hotstuff.Committee instance and measures the time which the HotStuff's core logic -// spends in the hotstuff.Committee component, i.e. the time determining consensus committee +// CommitteeMetricsWrapper implements the hotstuff.DynamicCommittee interface. +// It wraps a hotstuff.DynamicCommittee instance and measures the time which the HotStuff's core logic +// spends in the hotstuff.DynamicCommittee component, i.e. the time determining consensus committee // relations. The measured time durations are reported as values for the // CommitteeProcessingDuration metric. type CommitteeMetricsWrapper struct { - committee hotstuff.Committee + committee hotstuff.DynamicCommittee metrics module.HotstuffMetrics } -var _ hotstuff.Committee = (*CommitteeMetricsWrapper)(nil) +var _ hotstuff.Replicas = (*CommitteeMetricsWrapper)(nil) +var _ hotstuff.DynamicCommittee = (*CommitteeMetricsWrapper)(nil) -func NewMetricsWrapper(committee hotstuff.Committee, metrics module.HotstuffMetrics) *CommitteeMetricsWrapper { +func NewMetricsWrapper(committee hotstuff.DynamicCommittee, metrics module.HotstuffMetrics) *CommitteeMetricsWrapper { return &CommitteeMetricsWrapper{ committee: committee, metrics: metrics, } } -func (w CommitteeMetricsWrapper) Identities(blockID flow.Identifier) (flow.IdentityList, error) { +func (w CommitteeMetricsWrapper) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) { processStart := time.Now() - identities, err := w.committee.Identities(blockID) + identities, err := w.committee.IdentitiesByBlock(blockID) w.metrics.CommitteeProcessingDuration(time.Since(processStart)) return identities, err } -func (w CommitteeMetricsWrapper) Identity(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { +func (w CommitteeMetricsWrapper) IdentityByBlock(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { processStart := time.Now() - identity, err := w.committee.Identity(blockID, participantID) + identity, err := w.committee.IdentityByBlock(blockID, participantID) + w.metrics.CommitteeProcessingDuration(time.Since(processStart)) + return identity, err +} + +func (w CommitteeMetricsWrapper) IdentitiesByEpoch(view uint64) (flow.IdentityList, error) { + processStart := time.Now() + identities, err := w.committee.IdentitiesByEpoch(view) + w.metrics.CommitteeProcessingDuration(time.Since(processStart)) + return identities, err +} + +func (w CommitteeMetricsWrapper) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.Identity, error) { + processStart := time.Now() + identity, err := w.committee.IdentityByEpoch(view, participantID) w.metrics.CommitteeProcessingDuration(time.Since(processStart)) return identity, err } @@ -49,6 +64,20 @@ func (w CommitteeMetricsWrapper) LeaderForView(view uint64) (flow.Identifier, er return id, err } +func (w CommitteeMetricsWrapper) QuorumThresholdForView(view uint64) (uint64, error) { + processStart := time.Now() + id, err := w.committee.QuorumThresholdForView(view) + w.metrics.CommitteeProcessingDuration(time.Since(processStart)) + return id, err +} + +func (w CommitteeMetricsWrapper) TimeoutThresholdForView(view uint64) (uint64, error) { + processStart := time.Now() + id, err := w.committee.TimeoutThresholdForView(view) + w.metrics.CommitteeProcessingDuration(time.Since(processStart)) + return id, err +} + func (w CommitteeMetricsWrapper) Self() flow.Identifier { processStart := time.Now() id := w.committee.Self() @@ -56,9 +85,9 @@ func (w CommitteeMetricsWrapper) Self() flow.Identifier { return id } -func (w CommitteeMetricsWrapper) DKG(blockID flow.Identifier) (hotstuff.DKG, error) { +func (w CommitteeMetricsWrapper) DKG(view uint64) (hotstuff.DKG, error) { processStart := time.Now() - dkg, err := w.committee.DKG(blockID) + dkg, err := w.committee.DKG(view) w.metrics.CommitteeProcessingDuration(time.Since(processStart)) return dkg, err } diff --git a/consensus/hotstuff/committees/static.go b/consensus/hotstuff/committees/static.go index c58fad2bc72..b95c6448dff 100644 --- a/consensus/hotstuff/committees/static.go +++ b/consensus/hotstuff/committees/static.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" @@ -42,14 +43,26 @@ type Static struct { dkg protocol.DKG } -func (s Static) Identities(_ flow.Identifier) (flow.IdentityList, error) { +func (s Static) IdentitiesByBlock(_ flow.Identifier) (flow.IdentityList, error) { return s.participants, nil } -func (s Static) Identity(_ flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { +func (s Static) IdentityByBlock(_ flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { identity, ok := s.participants.ByNodeID(participantID) if !ok { - return nil, fmt.Errorf("unknown partipant") + return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) + } + return identity, nil +} + +func (s Static) IdentitiesByEpoch(_ uint64) (flow.IdentityList, error) { + return s.participants, nil +} + +func (s Static) IdentityByEpoch(_ uint64, participantID flow.Identifier) (*flow.Identity, error) { + identity, ok := s.participants.ByNodeID(participantID) + if !ok { + return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) } return identity, nil } @@ -58,11 +71,19 @@ func (s Static) LeaderForView(_ uint64) (flow.Identifier, error) { return flow.ZeroID, fmt.Errorf("invalid for static committee") } +func (s Static) QuorumThresholdForView(_ uint64) (uint64, error) { + return WeightThresholdToBuildQC(s.participants.TotalWeight()), nil +} + +func (s Static) TimeoutThresholdForView(_ uint64) (uint64, error) { + return WeightThresholdToTimeout(s.participants.TotalWeight()), nil +} + func (s Static) Self() flow.Identifier { return s.myID } -func (s Static) DKG(_ flow.Identifier) (hotstuff.DKG, error) { +func (s Static) DKG(_ uint64) (hotstuff.DKG, error) { return s.dkg, nil } diff --git a/consensus/hotstuff/committees/threshold.go b/consensus/hotstuff/committees/threshold.go new file mode 100644 index 00000000000..c68ef5f341c --- /dev/null +++ b/consensus/hotstuff/committees/threshold.go @@ -0,0 +1,25 @@ +package committees + +// WeightThresholdToBuildQC returns the weight (sum of unique, valid votes for this view) +// that is minimally required for a supermajority. +func WeightThresholdToBuildQC(totalWeight uint64) uint64 { + // Given totalWeight, we need the smallest integer t such that 2 * totalWeight / 3 < t + // Formally, the minimally required weight is: 2 * Floor(totalWeight/3) + max(1, totalWeight mod 3) + floorOneThird := totalWeight / 3 // integer division, includes floor + res := 2 * floorOneThird + divRemainder := totalWeight % 3 + if divRemainder <= 1 { + res = res + 1 + } else { + res += divRemainder + } + return res +} + +// WeightThresholdToTimeout returns the weight (sum of unique, valid timeout objects for this view) +// that is minimally required to immediately timeout and build a TO. +func WeightThresholdToTimeout(totalWeight uint64) uint64 { + // Given totalWeight, we need the smallest integer t such that totalWeight / 3 < t + // Formally, the minimally required weight is: Floor(totalWeight/3) + 1 + return totalWeight/3 + 1 +} diff --git a/consensus/hotstuff/committees/threshold_test.go b/consensus/hotstuff/committees/threshold_test.go new file mode 100644 index 00000000000..418d6d7f938 --- /dev/null +++ b/consensus/hotstuff/committees/threshold_test.go @@ -0,0 +1,32 @@ +package committees + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestComputeQCWeightThreshold tests computing the HotStuff safety threshold +// for producing a QC. +func TestComputeQCWeightThreshold(t *testing.T) { + // testing lowest values + for i := 1; i <= 302; i++ { + threshold := WeightThresholdToBuildQC(uint64(i)) + + boundaryValue := float64(i) * 2.0 / 3.0 + assert.True(t, boundaryValue < float64(threshold)) + assert.False(t, boundaryValue < float64(threshold-1)) + } +} + +// TestComputeTOWeightThreshold tests computing the HotStuff safety threshold +// for producing a TO. +func TestComputeTOWeightThreshold(t *testing.T) { + // testing lowest values + for totalWeight := uint64(1); totalWeight <= 302; totalWeight++ { + threshold := WeightThresholdToTimeout(totalWeight) + + assert.Greater(t, threshold*3, totalWeight) // 3*threshold > totalWeight + assert.LessOrEqual(t, (threshold-1)*3, totalWeight) // 3*(threshold-1) <= totalWeight + } +} diff --git a/consensus/hotstuff/communicator.go b/consensus/hotstuff/communicator.go deleted file mode 100644 index aed1ce215e9..00000000000 --- a/consensus/hotstuff/communicator.go +++ /dev/null @@ -1,24 +0,0 @@ -package hotstuff - -import ( - "time" - - "github.com/onflow/flow-go/model/flow" -) - -// Communicator is the component that allows the HotStuff core algorithm to -// communicate with the other actors of the consensus process. -type Communicator interface { - - // SendVote sends a vote for the given parameters to the specified recipient. - SendVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) error - - // BroadcastProposal broadcasts the given block proposal to all actors of - // the consensus process. - BroadcastProposal(proposal *flow.Header) error - - // BroadcastProposalWithDelay broadcasts the given block proposal to all actors of - // the consensus process. - // delay is to hold the proposal before broadcasting it. Useful to control the block production rate. - BroadcastProposalWithDelay(proposal *flow.Header, delay time.Duration) error -} diff --git a/consensus/hotstuff/consumer.go b/consensus/hotstuff/consumer.go index e57a39d110d..5eb592b9912 100644 --- a/consensus/hotstuff/consumer.go +++ b/consensus/hotstuff/consumer.go @@ -1,6 +1,8 @@ package hotstuff import ( + "time" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) @@ -49,6 +51,7 @@ type FinalizationConsumer interface { // - handle repetition of the same events (with some processing overhead). type Consumer interface { FinalizationConsumer + CommunicatorConsumer // OnEventProcessed notifications are produced by the EventHandler when it is done processing // and hands control back to the EventLoop to wait for the next event. @@ -57,11 +60,12 @@ type Consumer interface { // and must handle repetition of the same events (with some processing overhead). OnEventProcessed() - // OnReceiveVote notifications are produced by the EventHandler when it starts processing a vote. + // OnStart notifications are produced by the EventHandler when it starts blocks recovery and + // prepares for handling incoming events from EventLoop. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnReceiveVote(currentView uint64, vote *model.Vote) + OnStart(currentView uint64) // OnReceiveProposal notifications are produced by the EventHandler when it starts processing a block. // Prerequisites: @@ -69,39 +73,57 @@ type Consumer interface { // and must handle repetition of the same events (with some processing overhead). OnReceiveProposal(currentView uint64, proposal *model.Proposal) - // OnEnteringView notifications are produced by the EventHandler when it enters a new view. + // OnReceiveQc notifications are produced by the EventHandler when it starts processing a + // QuorumCertificate [QC] constructed by the node's internal vote aggregator. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnEnteringView(viewNumber uint64, leader flow.Identifier) + OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) - // OnQcTriggeredViewChange notifications are produced by PaceMaker when it moves to a new view - // based on processing a QC. The arguments specify the qc (first argument), which triggered - // the view change, and the newView to which the PaceMaker transitioned (second argument). + // OnReceiveTc notifications are produced by the EventHandler when it starts processing a + // TimeoutCertificate [TC] constructed by the node's internal timeout aggregator. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) + OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) - // OnProposingBlock notifications are produced by the EventHandler when the replica, as - // leader for the respective view, proposing a block. + // OnPartialTc notifications are produced by the EventHandler when it starts processing partial TC + // constructed by local timeout aggregator. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnProposingBlock(proposal *model.Proposal) + OnPartialTc(currentView uint64, partialTc *PartialTcCreated) - // OnVoting notifications are produced by the EventHandler when the replica votes for a block. + // OnLocalTimeout notifications are produced by the EventHandler when it reacts to expiry of round duration timer. + // Such a notification indicates that the PaceMaker's timeout was processed by the system. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnVoting(vote *model.Vote) + OnLocalTimeout(currentView uint64) - // OnQcConstructedFromVotes notifications are produced by the VoteAggregator - // component, whenever it constructs a QC from votes. + // OnViewChange notifications are produced by PaceMaker when it transitions to a new view + // based on processing a QC or TC. The arguments specify the oldView (first argument), + // and the newView to which the PaceMaker transitioned (second argument). // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) + OnViewChange(oldView, newView uint64) + + // OnQcTriggeredViewChange notifications are produced by PaceMaker when it moves to a new view + // based on processing a QC. The arguments specify the qc (first argument), which triggered + // the view change, and the newView to which the PaceMaker transitioned (second argument). + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) + + // OnTcTriggeredViewChange notifications are produced by PaceMaker when it moves to a new view + // based on processing a TC. The arguments specify the tc (first argument), which triggered + // the view change, and the newView to which the PaceMaker transitioned (second argument). + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) // OnStartingTimeout notifications are produced by PaceMaker. Such a notification indicates that the // PaceMaker is now waiting for the system to (receive and) process blocks or votes. @@ -109,29 +131,37 @@ type Consumer interface { // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnStartingTimeout(*model.TimerInfo) + OnStartingTimeout(model.TimerInfo) - // OnReachedTimeout notifications are produced by PaceMaker. Such a notification indicates that the - // PaceMaker's timeout was processed by the system. The specific timeout type is contained in the TimerInfo. + // OnVoteProcessed notifications are produced by the Vote Aggregation logic, each time + // we successfully ingest a valid vote. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnReachedTimeout(timeout *model.TimerInfo) + OnVoteProcessed(vote *model.Vote) - // OnQcIncorporated notifications are produced by ForkChoice - // whenever a quorum certificate is incorporated into the consensus state. + // OnTimeoutProcessed notifications are produced by the Timeout Aggregation logic, + // each time we successfully ingest a valid timeout. // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnQcIncorporated(*flow.QuorumCertificate) + OnTimeoutProcessed(timeout *model.TimeoutObject) - // OnForkChoiceGenerated notifications are produced by ForkChoice whenever a fork choice is generated. - // The arguments specify the view (first argument) of the block which is to be built and the - // quorum certificate (second argument) that is supposed to be in the block. + // OnCurrentViewDetails notifications are produced by the EventHandler during the course of a view with auxiliary information. + // These notifications are generally not produced for all views (for example skipped views). + // These notifications are guaranteed to be produced for all views we enter after fully processing a message. + // Example 1: + // - We are in view 8. We process a QC with view 10, causing us to enter view 11. + // - Then this notification will be produced for view 11. + // Example 2: + // - We are in view 8. We process a proposal with view 10, which contains a TC for view 9 and TC.NewestQC for view 8. + // - The QC would allow us to enter view 9 and the TC would allow us to enter view 10, + // so after fully processing the message we are in view 10. + // - Then this notification will be produced for view 10, but not view 9 // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnForkChoiceGenerated(uint64, *flow.QuorumCertificate) + OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) // OnDoubleVotingDetected notifications are produced by the Vote Aggregation logic // whenever a double voting (same voter voting for different blocks at the same view) was detected. @@ -145,7 +175,7 @@ type Consumer interface { // Prerequisites: // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). - OnInvalidVoteDetected(*model.Vote) + OnInvalidVoteDetected(err model.InvalidVoteError) // OnVoteForInvalidBlockDetected notifications are produced by the Vote Aggregation logic // whenever vote for invalid proposal was detected. @@ -153,6 +183,20 @@ type Consumer interface { // Implementation must be concurrency safe; Non-blocking; // and must handle repetition of the same events (with some processing overhead). OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) + + // OnDoubleTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever a double timeout (same replica producing two different timeouts at the same view) was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) + + // OnInvalidTimeoutDetected notifications are produced by the Timeout Aggregation logic + // whenever an invalid timeout was detected. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnInvalidTimeoutDetected(err model.InvalidTimeoutError) } // QCCreatedConsumer consumes outbound notifications produced by HotStuff and its components. @@ -172,3 +216,79 @@ type QCCreatedConsumer interface { // and must handle repetition of the same events (with some processing overhead). OnQcConstructedFromVotes(*flow.QuorumCertificate) } + +// TimeoutCollectorConsumer consumes outbound notifications produced by HotStuff's timeout aggregation +// component. These events are primarily intended for the HotStuff-internal state machine (EventHandler), +// but might also be relevant to the larger node in which HotStuff is running. +// +// Caution: the events are not strictly ordered by increasing views! +// The notifications are emitted by concurrent processing logic. Over larger time scales, the +// emitted events are for statistically increasing views. However, on short time scales there +// are _no_ monotonicity guarantees w.r.t. the events' views. +// +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type TimeoutCollectorConsumer interface { + // OnTcConstructedFromTimeouts notifications are produced by the TimeoutProcessor + // component, whenever it constructs a TC based on TimeoutObjects from a + // supermajority of consensus participants. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertificate) + + // OnPartialTcCreated notifications are produced by the TimeoutProcessor + // component, whenever it collected TimeoutObjects from a superminority + // of consensus participants for a specific view. Along with the view, it + // reports the newest QC and TC (for previous view) discovered in process of + // timeout collection. Per convention, the newest QC is never nil, while + // the TC for the previous view might be nil. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) + + // OnNewQcDiscovered notifications are produced by the TimeoutCollector + // component, whenever it discovers new QC included in timeout object. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnNewQcDiscovered(certificate *flow.QuorumCertificate) + + // OnNewTcDiscovered notifications are produced by the TimeoutCollector + // component, whenever it discovers new TC included in timeout object. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnNewTcDiscovered(certificate *flow.TimeoutCertificate) +} + +// CommunicatorConsumer consumes outbound notifications produced by HotStuff and it's components. +// Notifications allow the HotStuff core algorithm to communicate with the other actors of the consensus process. +// Implementations must: +// - be concurrency safe +// - be non-blocking +// - handle repetition of the same events (with some processing overhead). +type CommunicatorConsumer interface { + // OnOwnVote notifies about intent to send a vote for the given parameters to the specified recipient. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) + + // OnOwnTimeout notifies about intent to broadcast the given timeout object(TO) to all actors of the consensus process. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnOwnTimeout(timeout *model.TimeoutObject) + + // OnOwnProposal notifies about intent to broadcast the given block proposal to all actors of + // the consensus process. + // delay is to hold the proposal before broadcasting it. Useful to control the block production rate. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) +} diff --git a/consensus/hotstuff/event_handler.go b/consensus/hotstuff/event_handler.go index 654e38a12ec..a2134680389 100644 --- a/consensus/hotstuff/event_handler.go +++ b/consensus/hotstuff/event_handler.go @@ -1,28 +1,62 @@ package hotstuff import ( + "context" "time" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) +// PartialTcCreated represents a notification emitted by the TimeoutProcessor component, +// whenever it has collected TimeoutObjects from a superminority +// of consensus participants for a specific view. Along with the view, it +// reports the newest QC and TC (for previous view) discovered during +// timeout collection. Per convention, the newest QC is never nil, while +// the TC for the previous view might be nil. +type PartialTcCreated struct { + View uint64 + NewestQC *flow.QuorumCertificate + LastViewTC *flow.TimeoutCertificate +} + // EventHandler runs a state machine to process proposals, QC and local timeouts. +// Not concurrency safe. type EventHandler interface { - // OnQCConstructed processes a valid qc constructed by internal vote aggregator. - OnQCConstructed(qc *flow.QuorumCertificate) error + // OnReceiveQc processes a valid qc constructed by internal vote aggregator or discovered in TimeoutObject. + // All inputs should be validated before feeding into this function. Assuming trusted data. + // No errors are expected during normal operation. + OnReceiveQc(qc *flow.QuorumCertificate) error + + // OnReceiveTc processes a valid tc constructed by internal timeout aggregator, discovered in TimeoutObject or + // broadcast over the network. + // All inputs should be validated before feeding into this function. Assuming trusted data. + // No errors are expected during normal operation. + OnReceiveTc(tc *flow.TimeoutCertificate) error // OnReceiveProposal processes a block proposal received from another HotStuff // consensus participant. + // All inputs should be validated before feeding into this function. Assuming trusted data. + // No errors are expected during normal operation. OnReceiveProposal(proposal *model.Proposal) error - // OnLocalTimeout will check if there was a local timeout. + // OnLocalTimeout handles a local timeout event by creating a model.TimeoutObject and broadcasting it. + // No errors are expected during normal operation. OnLocalTimeout() error - // TimeoutChannel returs a channel that sends a signal on timeout. + // OnPartialTcCreated handles notification produces by the internal timeout aggregator. If the notification is for the current view, + // a corresponding model.TimeoutObject is broadcast to the consensus committee. + // No errors are expected during normal operation. + OnPartialTcCreated(partialTC *PartialTcCreated) error + + // TimeoutChannel returns a channel that sends a signal on timeout. TimeoutChannel() <-chan time.Time // Start starts the event handler. - Start() error + // No errors are expected during normal operation. + // CAUTION: EventHandler is not concurrency safe. The Start method must + // be executed by the same goroutine that also calls the other business logic + // methods, or concurrency safety has to be implemented externally. + Start(ctx context.Context) error } diff --git a/consensus/hotstuff/event_loop.go b/consensus/hotstuff/event_loop.go index ccf1b6fc4d7..f107449c457 100644 --- a/consensus/hotstuff/event_loop.go +++ b/consensus/hotstuff/event_loop.go @@ -1,15 +1,12 @@ package hotstuff import ( - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) // EventLoop performs buffer and processing of incoming proposals and QCs. type EventLoop interface { module.HotStuff - - // SubmitTrustedQC accepts QC for processing. QC will be dispatched on worker thread. - // CAUTION: QC is trusted (_not_ validated again), as it's built by ourselves. - SubmitTrustedQC(qc *flow.QuorumCertificate) + TimeoutCollectorConsumer + QCCreatedConsumer } diff --git a/consensus/hotstuff/eventhandler/event_handler.go b/consensus/hotstuff/eventhandler/event_handler.go index b47fb3202ca..e1558d64144 100644 --- a/consensus/hotstuff/eventhandler/event_handler.go +++ b/consensus/hotstuff/eventhandler/event_handler.go @@ -1,6 +1,7 @@ package eventhandler import ( + "context" "errors" "fmt" "time" @@ -10,25 +11,39 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/mempool" ) // EventHandler is the main handler for individual events that trigger state transition. -// It exposes API to handle one event at a time synchronously. The caller is -// responsible for running the event loop to ensure that. +// It exposes API to handle one event at a time synchronously. EventHandler is *not concurrency safe*. +// Please use the EventLoop to ensure that only a single go-routine executes the EventHandler's algorithms. +// EventHandler is implemented in event-driven way, it reacts to incoming events and performs certain actions. +// It doesn't perform any actions on its own. There are 3 main responsibilities of EventHandler, vote, propose, +// timeout. There are specific scenarios that lead to each of those actions. +// - create vote: voting logic is triggered by OnReceiveProposal, after receiving proposal we have all required information +// to create a valid vote. Compliance engine makes sure that we receive proposals, whose parents are known. +// Creating a vote can be triggered ONLY by receiving proposal. +// - create timeout: creating model.TimeoutObject[TO] is triggered by OnLocalTimeout, after reaching deadline for current round. +// EventHandler gets notified about it and has to create a model.TimeoutObject and broadcast it to other replicas. +// Creating a TO can be triggered by reaching round deadline or triggered as part of Bracha broadcast when superminority +// of replicas have contributed to TC creation and created a partial TC. +// - create a proposal: proposing logic is more complicated. Creating a proposal is triggered by the EventHandler receiving +// a QC or TC that induces a view change to a view where the replica is primary. As an edge case, the EventHandler +// can receive a QC or TC that triggers the view change, but we can't create a proposal in case we are missing parent block the newest QC refers to. +// In case we already have the QC, but are still missing the respective parent, OnReceiveProposal can trigger the proposing logic +// as well, but only when receiving proposal for view lower than active view. +// To summarize, to make a valid proposal for view N we need to have a QC or TC for N-1 and know the proposal with blockID +// NewestQC.BlockID. +// +// Not concurrency safe. type EventHandler struct { - log zerolog.Logger - paceMaker hotstuff.PaceMaker - blockProducer hotstuff.BlockProducer - forks hotstuff.Forks - persist hotstuff.Persister - communicator hotstuff.Communicator - committee hotstuff.Committee - voteAggregator hotstuff.VoteAggregator - voter hotstuff.Voter - validator hotstuff.Validator - notifier hotstuff.Consumer - ownProposal flow.Identifier + log zerolog.Logger + paceMaker hotstuff.PaceMaker + blockProducer hotstuff.BlockProducer + forks hotstuff.Forks + persist hotstuff.Persister + committee hotstuff.Replicas + safetyRules hotstuff.SafetyRules + notifier hotstuff.Consumer } var _ hotstuff.EventHandler = (*EventHandler)(nil) @@ -40,62 +55,88 @@ func NewEventHandler( blockProducer hotstuff.BlockProducer, forks hotstuff.Forks, persist hotstuff.Persister, - communicator hotstuff.Communicator, - committee hotstuff.Committee, - voteAggregator hotstuff.VoteAggregator, - voter hotstuff.Voter, - validator hotstuff.Validator, + committee hotstuff.Replicas, + safetyRules hotstuff.SafetyRules, notifier hotstuff.Consumer, ) (*EventHandler, error) { e := &EventHandler{ - log: log.With().Str("hotstuff", "participant").Logger(), - paceMaker: paceMaker, - blockProducer: blockProducer, - forks: forks, - persist: persist, - communicator: communicator, - voter: voter, - validator: validator, - committee: committee, - voteAggregator: voteAggregator, - notifier: notifier, - ownProposal: flow.ZeroID, + log: log.With().Str("hotstuff", "participant").Logger(), + paceMaker: paceMaker, + blockProducer: blockProducer, + forks: forks, + persist: persist, + safetyRules: safetyRules, + committee: committee, + notifier: notifier, } return e, nil } -// OnQCConstructed processes constructed QC by our vote aggregator -func (e *EventHandler) OnQCConstructed(qc *flow.QuorumCertificate) error { +// OnReceiveQc processes a valid qc constructed by internal vote aggregator or discovered in TimeoutObject. +// All inputs should be validated before feeding into this function. Assuming trusted data. +// No errors are expected during normal operation. +func (e *EventHandler) OnReceiveQc(qc *flow.QuorumCertificate) error { curView := e.paceMaker.CurView() - log := e.log.With(). Uint64("cur_view", curView). Uint64("qc_view", qc.View). Hex("qc_block_id", qc.BlockID[:]). Logger() - - e.notifier.OnQcConstructedFromVotes(curView, qc) + log.Debug().Msg("received QC") + e.notifier.OnReceiveQc(curView, qc) defer e.notifier.OnEventProcessed() - log.Debug().Msg("received constructed QC") + newViewEvent, err := e.paceMaker.ProcessQC(qc) + if err != nil { + return fmt.Errorf("could not process QC: %w", err) + } + if newViewEvent == nil { + log.Debug().Msg("QC didn't trigger view change, nothing to do") + return nil + } - // ignore stale qc - if qc.View < e.forks.FinalizedView() { - log.Debug().Msg("stale qc") + // current view has changed, go to new view + log.Debug().Msg("QC triggered view change, starting new view now") + return e.proposeForNewViewIfPrimary() +} + +// OnReceiveTc processes a valid tc constructed by internal timeout aggregator, discovered in TimeoutObject or +// broadcast over the network. +// All inputs should be validated before feeding into this function. Assuming trusted data. +// No errors are expected during normal operation. +func (e *EventHandler) OnReceiveTc(tc *flow.TimeoutCertificate) error { + curView := e.paceMaker.CurView() + log := e.log.With(). + Uint64("cur_view", curView). + Uint64("tc_view", tc.View). + Uint64("tc_newest_qc_view", tc.NewestQC.View). + Hex("tc_newest_qc_block_id", tc.NewestQC.BlockID[:]). + Logger() + log.Debug().Msg("received TC") + e.notifier.OnReceiveTc(curView, tc) + defer e.notifier.OnEventProcessed() + + newViewEvent, err := e.paceMaker.ProcessTC(tc) + if err != nil { + return fmt.Errorf("could not process TC for view %d: %w", tc.View, err) + } + if newViewEvent == nil { + log.Debug().Msg("TC didn't trigger view change, nothing to do") return nil } - return e.processQC(qc) + // current view has changed, go to new view + log.Debug().Msg("TC triggered view change, starting new view now") + return e.proposeForNewViewIfPrimary() } -// OnReceiveProposal processes the block when a block proposal is received. -// It is assumed that the block proposal is incorporated. (its parent can be found -// in the forks) +// OnReceiveProposal processes a block proposal received from another HotStuff +// consensus participant. +// All inputs should be validated before feeding into this function. Assuming trusted data. +// No errors are expected during normal operation. func (e *EventHandler) OnReceiveProposal(proposal *model.Proposal) error { - block := proposal.Block curView := e.paceMaker.CurView() - log := e.log.With(). Uint64("cur_view", curView). Uint64("block_view", block.View). @@ -103,10 +144,9 @@ func (e *EventHandler) OnReceiveProposal(proposal *model.Proposal) error { Uint64("qc_view", block.QC.View). Hex("proposer_id", block.ProposerID[:]). Logger() - + log.Debug().Msg("proposal received from compliance engine") e.notifier.OnReceiveProposal(curView, proposal) defer e.notifier.OnEventProcessed() - log.Debug().Msg("proposal forwarded from compliance engine") // ignore stale proposals if block.View < e.forks.FinalizedView() { @@ -114,66 +154,35 @@ func (e *EventHandler) OnReceiveProposal(proposal *model.Proposal) error { return nil } - // we skip validation for our last own proposal - if proposal.Block.BlockID != e.ownProposal { - - // validate the block. exit if the proposal is invalid - err := e.validator.ValidateProposal(proposal) - if model.IsInvalidBlockError(err) { - perr := e.voteAggregator.InvalidBlock(proposal) - if mempool.IsDecreasingPruningHeightError(perr) { - log.Warn().Err(err).Msgf("invalid block proposal, but vote aggregator has pruned this height: %v", perr) - return nil - } - - if perr != nil { - return fmt.Errorf("vote aggregator could not process invalid block proposal %v, err %v: %w", - block.BlockID, err, perr) - } - - log.Warn().Err(err).Msg("invalid block proposal") - return nil - } - - if errors.Is(err, model.ErrUnverifiableBlock) { - log.Warn().Err(err).Msg("unverifiable block proposal") - - // even if the block is unverifiable because the QC has been - // pruned, it still needs to be added to the forks, otherwise, - // a new block with a QC to this block will fail to be added - // to forks and crash the event loop. - } else if err != nil { - return fmt.Errorf("cannot validate proposal (%x): %w", block.BlockID, err) - } - } - - // notify vote aggregator about a new block, so that it can start verifying - // votes for it. - err := e.voteAggregator.AddBlock(proposal) + // store the block. + err := e.forks.AddProposal(proposal) if err != nil { - if !mempool.IsDecreasingPruningHeightError(err) { - return fmt.Errorf("could not add block (%v) to vote aggregator: %w", block.BlockID, err) - } + return fmt.Errorf("cannot add proposal to forks (%x): %w", block.BlockID, err) } - // store the block. - err = e.forks.AddBlock(block) + _, err = e.paceMaker.ProcessQC(proposal.Block.QC) if err != nil { - return fmt.Errorf("cannot add block to fork (%x): %w", block.BlockID, err) + return fmt.Errorf("could not process QC for block %x: %w", block.BlockID, err) } - // if the block is not for the current view, then process the QC - if block.View != curView { - return e.processQC(block.QC) + _, err = e.paceMaker.ProcessTC(proposal.LastViewTC) + if err != nil { + return fmt.Errorf("could not process TC for block %x: %w", block.BlockID, err) } - // if the block is for the current view, then check whether to vote for this block - err = e.processBlockForCurrentView(block) + // if the block is for the current view, then try voting for this block + err = e.processBlockForCurrentView(proposal) if err != nil { return fmt.Errorf("failed processing current block: %w", err) } + log.Debug().Msg("proposal processed from compliance engine") - return nil + // nothing to do if this proposal is for current view + if proposal.Block.View == e.paceMaker.CurView() { + return nil + } + + return e.proposeForNewViewIfPrimary() } // TimeoutChannel returns the channel for subscribing the waiting timeout on receiving @@ -182,190 +191,309 @@ func (e *EventHandler) TimeoutChannel() <-chan time.Time { return e.paceMaker.TimeoutChannel() } -// OnLocalTimeout is called when the timeout event created by pacemaker looped through the -// event loop. +// OnLocalTimeout handles a local timeout event by creating a model.TimeoutObject and broadcasting it. +// No errors are expected during normal operation. func (e *EventHandler) OnLocalTimeout() error { - curView := e.paceMaker.CurView() - newView := e.paceMaker.OnTimeout() + e.log.Debug().Uint64("cur_view", curView).Msg("timeout received from event loop") + e.notifier.OnLocalTimeout(curView) defer e.notifier.OnEventProcessed() - log := e.log.With(). + err := e.broadcastTimeoutObjectIfAuthorized() + if err != nil { + return fmt.Errorf("unexpected exception while processing timeout in view %d: %w", curView, err) + } + return nil +} + +// OnPartialTcCreated handles notification produces by the internal timeout aggregator. +// If the notification is for the current view, a corresponding model.TimeoutObject is broadcast to the consensus committee. +// No errors are expected during normal operation. +func (e *EventHandler) OnPartialTcCreated(partialTC *hotstuff.PartialTcCreated) error { + curView := e.paceMaker.CurView() + lastViewTC := partialTC.LastViewTC + logger := e.log.With(). Uint64("cur_view", curView). - Uint64("new_view", newView.View). - Logger() - // notifications about time-outs and view-changes are generated by PaceMaker; no need to send a notification here - log.Debug().Msg("timeout received from event loop") + Uint64("qc_view", partialTC.NewestQC.View) + if lastViewTC != nil { + logger.Uint64("last_view_tc_view", lastViewTC.View) + } + log := logger.Logger() + log.Debug().Msg("constructed partial TC") + + e.notifier.OnPartialTc(curView, partialTC) + defer e.notifier.OnEventProcessed() - if curView == newView.View { - return fmt.Errorf("OnLocalTimeout should guarantee that the pacemaker should go to next view, but didn't: (curView: %v, newView: %v)", curView, newView.View) + // process QC, this might trigger view change + _, err := e.paceMaker.ProcessQC(partialTC.NewestQC) + if err != nil { + return fmt.Errorf("could not process newest QC: %w", err) } - // current view has changed, go to new view - err := e.startNewView() + // process TC, this might trigger view change + _, err = e.paceMaker.ProcessTC(lastViewTC) if err != nil { - return fmt.Errorf("could not start new view: %w", err) + return fmt.Errorf("could not process TC for view %d: %w", lastViewTC.View, err) } - log.Debug().Msg("local timeout processed") + // NOTE: in other cases when we have observed a view change we will trigger proposing logic, this is desired logic + // for handling proposal, QC and TC. However, observing a partial TC means + // that superminority have timed out and there was at least one honest replica in that set. Honest replicas will never vote + // after timing out for current view meaning we won't be able to collect supermajority of votes for a proposal made after + // observing partial TC. - return nil -} + // by definition, we are allowed to produce timeout object if we have received partial TC for current view + if e.paceMaker.CurView() != partialTC.View { + return nil + } -// Start will start the pacemaker's timer and start the new view -func (e *EventHandler) Start() error { - e.paceMaker.Start() - return e.startNewView() + log.Debug().Msg("partial TC generated for current view, broadcasting timeout") + err = e.broadcastTimeoutObjectIfAuthorized() + if err != nil { + return fmt.Errorf("unexpected exception while processing partial TC in view %d: %w", partialTC.View, err) + } + return nil } -// startNewView will only be called when there is a view change from pacemaker. -// It reads the current view, and check if it needs to propose or vote in this view. -func (e *EventHandler) startNewView() error { +// Start starts the event handler. +// No errors are expected during normal operation. +// CAUTION: EventHandler is not concurrency safe. The Start method must +// be executed by the same goroutine that also calls the other business logic +// methods, or concurrency safety has to be implemented externally. +func (e *EventHandler) Start(ctx context.Context) error { + // notify about commencing recovery procedure + e.notifier.OnStart(e.paceMaker.CurView()) + defer e.notifier.OnEventProcessed() + e.paceMaker.Start(ctx) - // track the start time - start := time.Now() + err := e.processPendingBlocks() + if err != nil { + return fmt.Errorf("could not process pending blocks: %w", err) + } + err = e.proposeForNewViewIfPrimary() + if err != nil { + return fmt.Errorf("could not start new view: %w", err) + } + return nil +} +// broadcastTimeoutObjectIfAuthorized attempts to generate a model.TimeoutObject, adds it +// to `timeoutAggregator` and broadcasts it to the consensus commettee. We check, whether +// this node, at the current view, is part of the consensus committee. Otherwise, this +// method is functionally a no-op. +// For example, right after an epoch switchover a consensus node might still be online but +// not part of the _active_ consensus committee anymore. Consequently, it should not broadcast +// timeouts anymore. +// No errors are expected during normal operation. +func (e *EventHandler) broadcastTimeoutObjectIfAuthorized() error { curView := e.paceMaker.CurView() + newestQC := e.paceMaker.NewestQC() + lastViewTC := e.paceMaker.LastViewTC() + log := e.log.With().Uint64("cur_view", curView).Logger() + + if newestQC.View+1 == curView { + // in case last view has ended with QC and TC, make sure that only QC is included + // otherwise such timeout is invalid. This case is possible if TC has included QC with the same + // view as the TC itself, meaning that newestQC.View == lastViewTC.View + lastViewTC = nil + } - err := e.persist.PutStarted(curView) + timeout, err := e.safetyRules.ProduceTimeout(curView, newestQC, lastViewTC) if err != nil { - return fmt.Errorf("could not persist current view: %w", err) + if model.IsNoTimeoutError(err) { + log.Warn().Err(err).Msgf("not generating timeout as this node is not part of the active committee") + return nil + } + return fmt.Errorf("could not produce timeout: %w", err) } + // raise a notification to broadcast timeout + e.notifier.OnOwnTimeout(timeout) + log.Debug().Msg("broadcast TimeoutObject done") + + return nil +} + +// processPendingBlocks performs processing of pending blocks that were applied to chain state but weren't processed +// by Hotstuff event loop. Due to asynchronous nature of our processing pipelines compliance engine can validate and apply +// blocks to the chain state but fail to deliver them to EventHandler because of shutdown or crash. To recover those QCs and TCs +// recovery logic puts them in Forks and EventHandler can traverse pending blocks by view to obtain them. +func (e *EventHandler) processPendingBlocks() error { + newestView := e.forks.NewestView() + currentView := e.paceMaker.CurView() + for { + paceMakerActiveView := e.paceMaker.CurView() + if currentView < paceMakerActiveView { + currentView = paceMakerActiveView + } + + if currentView > newestView { + return nil + } + + // check if there are pending proposals for active view + pendingProposals := e.forks.GetProposalsForView(currentView) + // process all proposals for view, we are dealing only with valid QCs and TCs so no harm in processing + // double proposals here. + for _, proposal := range pendingProposals { + block := proposal.Block + _, err := e.paceMaker.ProcessQC(block.QC) + if err != nil { + return fmt.Errorf("could not process QC for block %x: %w", block.BlockID, err) + } + + _, err = e.paceMaker.ProcessTC(proposal.LastViewTC) + if err != nil { + return fmt.Errorf("could not process TC for block %x: %w", block.BlockID, err) + } + + // TODO(active-pacemaker): generally speaking we are only interested in QC and TC, but in some cases + // we might want to vote for blocks as well. Discuss if it's needed. + } + + currentView++ + } +} + +// proposeForNewViewIfPrimary will only be called when we may able to propose a block, after processing a new event. +// - after entering a new view as a result of processing a QC or TC, then we may propose for the newly entered view +// - after receiving a proposal (but not changing view), if that proposal is referenced by our highest known QC, +// and the proposal was previously unknown, then we can propose a block in the current view +// +// It reads the current view, and generates a proposal if we are the leader. +// No errors are expected during normal operation. +func (e *EventHandler) proposeForNewViewIfPrimary() error { + start := time.Now() // track the start time + curView := e.paceMaker.CurView() currentLeader, err := e.committee.LeaderForView(curView) if err != nil { return fmt.Errorf("failed to determine primary for new view %d: %w", curView, err) } - + finalizedView := e.forks.FinalizedView() log := e.log.With(). Uint64("cur_view", curView). + Uint64("finalized_view", finalizedView). Hex("leader_id", currentLeader[:]).Logger() - log.Debug(). - Uint64("finalized_view", e.forks.FinalizedView()). - Msg("entering new view") - e.notifier.OnEnteringView(curView, currentLeader) - - if e.committee.Self() == currentLeader { - log.Debug().Msg("generating block proposal as leader") - - // as the leader of the current view, - // build the block proposal for the current view - qc, _, err := e.forks.MakeForkChoice(curView) - if err != nil { - return fmt.Errorf("can not make fork choice for view %v: %w", curView, err) - } - proposal, err := e.blockProducer.MakeBlockProposal(qc, curView) - if err != nil { - return fmt.Errorf("can not make block proposal for curView %v: %w", curView, err) + e.notifier.OnCurrentViewDetails(curView, finalizedView, currentLeader) + + // check that I am the primary for this view and that I haven't already proposed; otherwise there is nothing to do + if e.committee.Self() != currentLeader { + return nil + } + for _, p := range e.forks.GetProposalsForView(curView) { + if p.Block.ProposerID == e.committee.Self() { + log.Debug().Msg("already proposed for current view") + return nil } - e.notifier.OnProposingBlock(proposal) - - block := proposal.Block - log.Debug(). - Uint64("block_view", block.View). - Hex("block_id", block.BlockID[:]). - Uint64("parent_view", qc.View). - Hex("parent_id", qc.BlockID[:]). - Hex("signer", block.ProposerID[:]). - Msg("forwarding proposal to communicator for broadcasting") - - // broadcast the proposal - header := model.ProposalToFlow(proposal) - delay := e.paceMaker.BlockRateDelay() - elapsed := time.Since(start) - if elapsed > delay { - delay = 0 - } else { - delay = delay - elapsed + } + + // attempt to generate proposal: + newestQC := e.paceMaker.NewestQC() + lastViewTC := e.paceMaker.LastViewTC() + + _, found := e.forks.GetProposal(newestQC.BlockID) + if !found { + // we don't know anything about block referenced by our newest QC, in this case we can't + // create a valid proposal since we can't guarantee validity of block payload. + log.Warn(). + Uint64("qc_view", newestQC.View). + Hex("block_id", newestQC.BlockID[:]).Msg("haven't synced the latest block yet; can't propose") + return nil + } + log.Debug().Msg("generating proposal as leader") + + // Sanity checks to make sure that resulting proposal is valid: + // In its proposal, the leader for view N needs to present evidence that it has legitimately entered view N. + // As evidence, we include a QC or TC for view N-1, which should always be available as the PaceMaker advances + // to view N only after observing a QC or TC from view N-1. Moreover QC and TC are always processed together. As + // EventHandler is strictly single-threaded without reentrancy, we must have a QC or TC for the prior view (curView-1). + // Failing one of these sanity checks is a symptom of state corruption or a severe implementation bug. + if newestQC.View+1 != curView { + if lastViewTC == nil { + return fmt.Errorf("possible state corruption, expected lastViewTC to be not nil") } - err = e.communicator.BroadcastProposalWithDelay(header, delay) - if err != nil { - log.Warn().Err(err).Msg("could not forward proposal") + if lastViewTC.View+1 != curView { + return fmt.Errorf("possible state corruption, don't have QC(view=%d) and TC(view=%d) for previous view(currentView=%d)", + newestQC.View, lastViewTC.View, curView) } - // mark our own proposals to avoid double validation - e.ownProposal = proposal.Block.BlockID + } else { + // In case last view has ended with QC and TC, make sure that only QC is included, + // otherwise such proposal is invalid. This case is possible if TC has included QC with the same + // view as the TC itself, meaning that newestQC.View == lastViewTC.View + lastViewTC = nil + } - // We return here to correspond to the HotStuff state machine. - return nil - // Algorithmically, this return statement is optional: - // * If this replica is the leader for the current view, there can be no valid proposal from any - // other node. This replica's proposal is the only valid proposal. - // * This replica's proposal got just sent out above. It will enter the HotStuff logic from the - // EventLoop. In other words, the own proposal is not yet stored in Forks. - // Hence, Forks cannot contain _any_ valid proposal for the current view. - // Therefore, if this replica is the leader, the following code is a no-op. - } - - // as a replica of the current view, find and process the block for the current view - blocks := e.forks.GetBlocksForView(curView) - if len(blocks) == 0 { - // if there is no block stored before for the current view, then exit and keep waiting - log.Debug().Msg("waiting for proposal from leader") - return nil + flowProposal, err := e.blockProducer.MakeBlockProposal(curView, newestQC, lastViewTC) + if err != nil { + return fmt.Errorf("can not make block proposal for curView %v: %w", curView, err) } + proposal := model.ProposalFromFlow(flowProposal) // turn the signed flow header into a proposal - // when there are multiple block proposals, we will just pick the first one. - // forks is responsible for slashing double proposal behavior, and - // event handler is aware of double proposals, but picking any should work and - // won't hurt safety - block := blocks[0] + // we want to store created proposal in forks to make sure that we don't create more proposals for + // current view. Due to asynchronous nature of our design it's possible that after creating proposal + // we will be asked to propose again for same view. + err = e.forks.AddProposal(proposal) + if err != nil { + return fmt.Errorf("could not add newly created proposal (%v): %w", proposal.Block.BlockID, err) + } + block := proposal.Block log.Debug(). Uint64("block_view", block.View). Hex("block_id", block.BlockID[:]). - Uint64("parent_view", block.QC.View). - Hex("parent_id", block.QC.BlockID[:]). + Uint64("parent_view", newestQC.View). + Hex("parent_id", newestQC.BlockID[:]). Hex("signer", block.ProposerID[:]). - Msg("processing cached proposal from leader") + Msg("forwarding proposal to communicator for broadcasting") - return e.processBlockForCurrentView(block) + // raise a notification with proposal (also triggers broadcast) + targetPublicationTime := start.Add(e.paceMaker.BlockRateDelay()) + e.notifier.OnOwnProposal(flowProposal, targetPublicationTime) + return nil } // processBlockForCurrentView processes the block for the current view. // It is called AFTER the block has been stored or found in Forks // It checks whether to vote for this block. -// It might trigger a view change to go to a different view, which might re-enter this function. -func (e *EventHandler) processBlockForCurrentView(block *model.Block) error { +// No errors are expected during normal operation. +func (e *EventHandler) processBlockForCurrentView(proposal *model.Proposal) error { // sanity check that block is really for the current view: curView := e.paceMaker.CurView() + block := proposal.Block if block.View != curView { - return fmt.Errorf("sanity check fails: block proposal's view does not match with curView, (blockView: %v, curView: %v)", - block.View, curView) + // ignore outdated proposals in case we have moved forward + return nil } // leader (node ID) for next view nextLeader, err := e.committee.LeaderForView(curView + 1) + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We are attempting process a block in an unknown epoch + // This should never happen, because: + // * the compliance layer ensures proposals are passed to the event loop strictly after their parent + // * the protocol state ensures that, before incorporating the first block of an epoch E, + // either E is known or we have triggered epoch fallback mode - in either case the epoch for the + // current epoch is known + return fmt.Errorf("attempting to process a block for current view in unknown epoch") + } if err != nil { return fmt.Errorf("failed to determine primary for next view %d: %w", curView+1, err) } - // voter performs all the checks to decide whether to vote for this block or not. - err = e.ownVote(block, curView, nextLeader) + // safetyRules performs all the checks to decide whether to vote for this block or not. + err = e.ownVote(proposal, curView, nextLeader) if err != nil { return fmt.Errorf("unexpected error in voting logic: %w", err) } - // Inform PaceMaker that we've processed a block for the current view. We expect - // a view change if an only if we are _not_ the next leader. We perform a sanity - // check here, because a wrong view change can have disastrous consequences. - isSelfNextLeader := e.committee.Self() == nextLeader - _, viewChanged := e.paceMaker.UpdateCurViewWithBlock(block, isSelfNextLeader) - if viewChanged == isSelfNextLeader { - if isSelfNextLeader { - return fmt.Errorf("I am primary for next view (%v) and should be collecting votes, but pacemaker triggered already view change", curView+1) - } - return fmt.Errorf("pacemaker should trigger a view change to net view (%v), but didn't", curView+1) - } - - if viewChanged { - return e.startNewView() - } return nil } // ownVote generates and forwards the own vote, if we decide to vote. // Any errors are potential symptoms of uncovered edge cases or corrupted internal state (fatal). -func (e *EventHandler) ownVote(block *model.Block, curView uint64, nextLeader flow.Identifier) error { +// No errors are expected during normal operation. +func (e *EventHandler) ownVote(proposal *model.Proposal, curView uint64, nextLeader flow.Identifier) error { + block := proposal.Block log := e.log.With(). Uint64("block_view", block.View). Hex("block_id", block.BlockID[:]). @@ -374,8 +502,16 @@ func (e *EventHandler) ownVote(block *model.Block, curView uint64, nextLeader fl Hex("signer", block.ProposerID[:]). Logger() - // voter performs all the checks to decide whether to vote for this block or not. - ownVote, err := e.voter.ProduceVoteIfVotable(block, curView) + _, found := e.forks.GetProposal(proposal.Block.QC.BlockID) + if !found { + // we don't have parent for this proposal, we can't vote since we can't guarantee validity of proposals + // payload. Strictly speaking this shouldn't ever happen because compliance engine makes sure that we + // receive proposals with valid parents. + return fmt.Errorf("won't vote for proposal, no parent block for this proposal") + } + + // safetyRules performs all the checks to decide whether to vote for this block or not. + ownVote, err := e.safetyRules.ProduceVote(proposal, curView) if err != nil { if !model.IsNoVoteError(err) { // unknown error, exit the event loop @@ -385,42 +521,8 @@ func (e *EventHandler) ownVote(block *model.Block, curView uint64, nextLeader fl return nil } - // The following code is only reached, if this replica has produced a vote. - // Send the vote to the next leader (or directly process it, if I am the next leader). - e.notifier.OnVoting(ownVote) log.Debug().Msg("forwarding vote to compliance engine") - if e.committee.Self() == nextLeader { // I am the next leader - e.voteAggregator.AddVote(ownVote) - } else { - err = e.communicator.SendVote(ownVote.BlockID, ownVote.View, ownVote.SigData, nextLeader) - if err != nil { - log.Warn().Err(err).Msg("could not forward vote") - } - } + // raise a notification to send vote + e.notifier.OnOwnVote(ownVote.BlockID, ownVote.View, ownVote.SigData, nextLeader) return nil } - -// processQC stores the QC and check whether the QC will trigger view change. -// If triggered, then go to the new view. -func (e *EventHandler) processQC(qc *flow.QuorumCertificate) error { - - log := e.log.With(). - Uint64("block_view", qc.View). - Hex("block_id", qc.BlockID[:]). - Logger() - - err := e.forks.AddQC(qc) - if err != nil { - return fmt.Errorf("cannot add QC to forks: %w", err) - } - - _, viewChanged := e.paceMaker.UpdateCurViewWithQC(qc) - if !viewChanged { - log.Debug().Msg("QC didn't trigger view change, nothing to do") - return nil - } - log.Debug().Msg("QC triggered view change, starting new view now") - - // current view has changed, go to new view - return e.startNewView() -} diff --git a/consensus/hotstuff/eventhandler/event_handler_test.go b/consensus/hotstuff/eventhandler/event_handler_test.go index 85eed9d1fc2..485b0cc91f2 100644 --- a/consensus/hotstuff/eventhandler/event_handler_test.go +++ b/consensus/hotstuff/eventhandler/event_handler_test.go @@ -1,6 +1,8 @@ package eventhandler import ( + "context" + "errors" "fmt" "os" "testing" @@ -13,389 +15,516 @@ import ( "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) const ( - startRepTimeout float64 = 400.0 // Milliseconds - minRepTimeout float64 = 100.0 // Milliseconds - voteTimeoutFraction float64 = 0.5 // multiplicative factor - multiplicativeIncrease float64 = 1.5 // multiplicative factor - multiplicativeDecrease float64 = 0.85 // multiplicative factor + minRepTimeout float64 = 100.0 // Milliseconds + maxRepTimeout float64 = 600.0 // Milliseconds + multiplicativeIncrease float64 = 1.5 // multiplicative factor + happyPathMaxRoundFailures uint64 = 6 // number of failed rounds before first timeout increase ) // TestPaceMaker is a real pacemaker module with logging for view changes type TestPaceMaker struct { hotstuff.PaceMaker - t require.TestingT } -func NewTestPaceMaker(t require.TestingT, startView uint64, timeoutController *timeout.Controller, notifier hotstuff.Consumer) *TestPaceMaker { - p, err := pacemaker.New(startView, timeoutController, notifier) +var _ hotstuff.PaceMaker = (*TestPaceMaker)(nil) + +func NewTestPaceMaker(timeoutController *timeout.Controller, + notifier hotstuff.Consumer, + persist hotstuff.Persister, +) *TestPaceMaker { + p, err := pacemaker.New(timeoutController, notifier, persist) if err != nil { panic(err) } - return &TestPaceMaker{p, t} + return &TestPaceMaker{p} } -func (p *TestPaceMaker) UpdateCurViewWithQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, bool) { +func (p *TestPaceMaker) ProcessQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, error) { oldView := p.CurView() - newView, changed := p.PaceMaker.UpdateCurViewWithQC(qc) - log.Info().Msgf("pacemaker.UpdateCurViewWithQC old view: %v, new view: %v\n", oldView, p.CurView()) - return newView, changed + newView, err := p.PaceMaker.ProcessQC(qc) + log.Info().Msgf("pacemaker.ProcessQC old view: %v, new view: %v\n", oldView, p.CurView()) + return newView, err } -func (p *TestPaceMaker) UpdateCurViewWithBlock(block *model.Block, isLeaderForNextView bool) (*model.NewViewEvent, bool) { +func (p *TestPaceMaker) ProcessTC(tc *flow.TimeoutCertificate) (*model.NewViewEvent, error) { oldView := p.CurView() - newView, changed := p.PaceMaker.UpdateCurViewWithBlock(block, isLeaderForNextView) - log.Info().Msgf("pacemaker.UpdateCurViewWithBlock old view: %v, new view: %v\n", oldView, p.CurView()) - return newView, changed + newView, err := p.PaceMaker.ProcessTC(tc) + log.Info().Msgf("pacemaker.ProcessTC old view: %v, new view: %v\n", oldView, p.CurView()) + return newView, err } -func (p *TestPaceMaker) OnTimeout() *model.NewViewEvent { - oldView := p.CurView() - newView := p.PaceMaker.OnTimeout() - log.Info().Msgf("pacemaker.OnTimeout old view: %v, new view: %v\n", oldView, p.CurView()) - return newView +func (p *TestPaceMaker) NewestQC() *flow.QuorumCertificate { + return p.PaceMaker.NewestQC() +} + +func (p *TestPaceMaker) LastViewTC() *flow.TimeoutCertificate { + return p.PaceMaker.LastViewTC() } // using a real pacemaker for testing event handler -func initPaceMaker(t require.TestingT, view uint64) hotstuff.PaceMaker { +func initPaceMaker(t require.TestingT, ctx context.Context, livenessData *hotstuff.LivenessData) hotstuff.PaceMaker { notifier := &mocks.Consumer{} tc, err := timeout.NewConfig( - time.Duration(startRepTimeout*1e6), time.Duration(minRepTimeout*1e6), - voteTimeoutFraction, + time.Duration(maxRepTimeout*1e6), multiplicativeIncrease, - multiplicativeDecrease, - 0) - if err != nil { - t.FailNow() - } - pm := NewTestPaceMaker(t, view, timeout.NewController(tc), notifier) + happyPathMaxRoundFailures, + 0, + time.Duration(maxRepTimeout*1e6)) + require.NoError(t, err) + persist := &mocks.Persister{} + persist.On("PutLivenessData", mock.Anything).Return(nil).Maybe() + persist.On("GetLivenessData").Return(livenessData, nil).Once() + pm := NewTestPaceMaker(timeout.NewController(tc), notifier, persist) notifier.On("OnStartingTimeout", mock.Anything).Return() - notifier.On("OnQcTriggeredViewChange", mock.Anything, mock.Anything).Return() - notifier.On("OnReachedTimeout", mock.Anything).Return() - pm.Start() + notifier.On("OnQcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return() + notifier.On("OnTcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return() + notifier.On("OnViewChange", mock.Anything, mock.Anything).Maybe() + pm.Start(ctx) return pm } -// VoteAggregator is a mock for testing eventhandler -type VoteAggregator struct { - // if a blockID exists in qcs field, then a vote can be made into a QC - qcs map[flow.Identifier]*flow.QuorumCertificate - t require.TestingT -} - -func NewVoteAggregator(t require.TestingT) *VoteAggregator { - return &VoteAggregator{ - qcs: make(map[flow.Identifier]*flow.QuorumCertificate), - t: t, - } -} - -func (v *VoteAggregator) StoreVoteAndBuildQC(vote *model.Vote, block *model.Block) (*flow.QuorumCertificate, bool, error) { - qc, ok := v.qcs[block.BlockID] - log.Info().Msgf("voteaggregator.StoreVoteAndBuildQC, qc built: %v, for view: %x, blockID: %v\n", ok, block.View, block.BlockID) - - return qc, ok, nil -} - -func (v *VoteAggregator) StorePendingVote(vote *model.Vote) (bool, error) { - return false, nil -} - -func (v *VoteAggregator) StoreProposerVote(vote *model.Vote) bool { - return true -} - -func (v *VoteAggregator) BuildQCOnReceivedBlock(block *model.Block) (*flow.QuorumCertificate, bool, error) { - qc, ok := v.qcs[block.BlockID] - log.Info().Msgf("voteaggregator.BuildQCOnReceivedBlock, qc built: %v, for view: %x, blockID: %v\n", ok, block.View, block.BlockID) - - return qc, ok, nil -} - -func (v *VoteAggregator) PruneByView(view uint64) { - log.Info().Msgf("pruned at view:%v\n", view) -} - +// Committee mocks hotstuff.DynamicCommittee and allows to easily control leader for some view. type Committee struct { - mocks.Committee + *mocks.Replicas // to mock I'm the leader of a certain view, add the view into the keys of leaders field leaders map[uint64]struct{} } -func NewCommittee() *Committee { - return &Committee{ - leaders: make(map[uint64]struct{}), +func NewCommittee(t *testing.T) *Committee { + committee := &Committee{ + Replicas: mocks.NewReplicas(t), + leaders: make(map[uint64]struct{}), } -} + self := unittest.IdentityFixture(unittest.WithNodeID(flow.Identifier{0x01})) + committee.On("LeaderForView", mock.Anything).Return(func(view uint64) flow.Identifier { + _, isLeader := committee.leaders[view] + if isLeader { + return self.NodeID + } + return flow.Identifier{0x00} + }, func(view uint64) error { + return nil + }).Maybe() -func (c *Committee) LeaderForView(view uint64) (flow.Identifier, error) { - _, isLeader := c.leaders[view] - if isLeader { - return flow.Identifier{0x01}, nil - } - return flow.Identifier{0x00}, nil -} + committee.On("Self").Return(self.NodeID).Maybe() -func (c *Committee) Self() flow.Identifier { - return flow.Identifier{0x01} + return committee } -// The Voter mock will not vote for any block unless the block's ID exists in votable field's key -type Voter struct { - votable map[flow.Identifier]struct{} - lastVotedView uint64 - t require.TestingT +// The SafetyRules mock will not vote for any block unless the block's ID exists in votable field's key +type SafetyRules struct { + *mocks.SafetyRules + votable map[flow.Identifier]struct{} } -func NewVoter(t require.TestingT, lastVotedView uint64) *Voter { - return &Voter{ - votable: make(map[flow.Identifier]struct{}), - lastVotedView: lastVotedView, - t: t, +func NewSafetyRules(t *testing.T) *SafetyRules { + safetyRules := &SafetyRules{ + SafetyRules: mocks.NewSafetyRules(t), + votable: make(map[flow.Identifier]struct{}), } -} -// voter will not vote for any block, unless the blockID exists in votable map -func (v *Voter) ProduceVoteIfVotable(block *model.Block, curView uint64) (*model.Vote, error) { - _, ok := v.votable[block.BlockID] - if !ok { - return nil, model.NoVoteError{Msg: "block not found"} - } - return createVote(block), nil + // SafetyRules will not vote for any block, unless the blockID exists in votable map + safetyRules.On("ProduceVote", mock.Anything, mock.Anything).Return( + func(block *model.Proposal, _ uint64) *model.Vote { + _, ok := safetyRules.votable[block.Block.BlockID] + if !ok { + return nil + } + return createVote(block.Block) + }, + func(block *model.Proposal, _ uint64) error { + _, ok := safetyRules.votable[block.Block.BlockID] + if !ok { + return model.NewNoVoteErrorf("block not found") + } + return nil + }).Maybe() + + safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return( + func(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) *model.TimeoutObject { + return helper.TimeoutObjectFixture(func(timeout *model.TimeoutObject) { + timeout.View = curView + timeout.NewestQC = newestQC + timeout.LastViewTC = lastViewTC + }) + }, + func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) error { return nil }).Maybe() + + return safetyRules } -// Forks mock allows to customize the Add QC and AddBlock function by specifying the addQC and addBlock callbacks +// Forks mock allows to customize the AddBlock function by specifying the addProposal callbacks type Forks struct { - mocks.Forks - // blocks stores all the blocks that have been added to the forks - blocks map[flow.Identifier]*model.Block + *mocks.Forks + // proposals stores all the proposals that have been added to the forks + proposals map[flow.Identifier]*model.Proposal finalized uint64 t require.TestingT - qc *flow.QuorumCertificate - // addQC is to customize the logic to change finalized view - addQC func(qc *flow.QuorumCertificate) error - // addBlock is to customize the logic to change finalized view - addBlock func(block *model.Block) error + // addProposal is to customize the logic to change finalized view + addProposal func(block *model.Proposal) error } -func NewForks(t require.TestingT, finalized uint64) *Forks { +func NewForks(t *testing.T, finalized uint64) *Forks { f := &Forks{ - blocks: make(map[flow.Identifier]*model.Block), + Forks: mocks.NewForks(t), + proposals: make(map[flow.Identifier]*model.Proposal), finalized: finalized, - t: t, } - f.addQC = func(qc *flow.QuorumCertificate) error { - if f.qc == nil || qc.View > f.qc.View { - f.qc = qc + f.On("AddProposal", mock.Anything).Return(func(proposal *model.Proposal) error { + log.Info().Msgf("forks.AddProposal received Proposal for view: %v, QC: %v\n", proposal.Block.View, proposal.Block.QC.View) + return f.addProposal(proposal) + }).Maybe() + + f.On("FinalizedView").Return(func() uint64 { + return f.finalized + }).Maybe() + + f.On("GetProposal", mock.Anything).Return(func(blockID flow.Identifier) *model.Proposal { + b := f.proposals[blockID] + return b + }, func(blockID flow.Identifier) bool { + b, ok := f.proposals[blockID] + var view uint64 + if ok { + view = b.Block.View } - return nil - } - f.addBlock = func(block *model.Block) error { - f.blocks[block.BlockID] = block + log.Info().Msgf("forks.GetProposal found %v: view: %v\n", ok, view) + return ok + }).Maybe() + + f.On("GetProposalsForView", mock.Anything).Return(func(view uint64) []*model.Proposal { + proposals := make([]*model.Proposal, 0) + for _, b := range f.proposals { + if b.Block.View == view { + proposals = append(proposals, b) + } + } + log.Info().Msgf("forks.GetProposalsForView found %v block(s) for view %v\n", len(proposals), view) + return proposals + }).Maybe() + + f.addProposal = func(proposal *model.Proposal) error { + block := proposal.Block + f.proposals[block.BlockID] = proposal if block.QC == nil { panic(fmt.Sprintf("block has no QC: %v", block.View)) } - _ = f.addQC(block.QC) return nil } - qc := createQC(createBlock(finalized)) - _ = f.addQC(qc) - return f } -func (f *Forks) AddBlock(block *model.Block) error { - log.Info().Msgf("forks.AddBlock received Block for view: %v, qc: %v\n", block.View, block.QC.View) - return f.addBlock(block) +// BlockProducer mock will always make a valid block +type BlockProducer struct { + proposerID flow.Identifier } -func (f *Forks) AddQC(qc *flow.QuorumCertificate) error { - log.Info().Msgf("forks.AddQC received QC for view: %v\n", qc.View) - return f.addQC(qc) +func (b *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*flow.Header, error) { + return model.ProposalToFlow(&model.Proposal{ + Block: helper.MakeBlock( + helper.WithBlockView(view), + helper.WithBlockQC(qc), + helper.WithBlockProposer(b.proposerID), + ), + LastViewTC: lastViewTC, + }), nil } -func (f *Forks) FinalizedView() uint64 { - return f.finalized +func TestEventHandler(t *testing.T) { + suite.Run(t, new(EventHandlerSuite)) } -func (f *Forks) GetBlock(blockID flow.Identifier) (*model.Block, bool) { - b, ok := f.blocks[blockID] - var view uint64 - if ok { - view = b.View - } - log.Info().Msgf("forks.GetBlock found: %v, view: %v\n", ok, view) - return b, ok -} +// EventHandlerSuite contains mocked state for testing event handler under different scenarios. +type EventHandlerSuite struct { + suite.Suite -func (f *Forks) GetBlocksForView(view uint64) []*model.Block { - blocks := make([]*model.Block, 0) - for _, b := range f.blocks { - if b.View == view { - blocks = append(blocks, b) - } - } - log.Info().Msgf("forks.GetBlocksForView found %v block(s) for view %v\n", len(blocks), view) - return blocks + eventhandler *EventHandler + + paceMaker hotstuff.PaceMaker + forks *Forks + persist *mocks.Persister + blockProducer *BlockProducer + committee *Committee + notifier *mocks.Consumer + safetyRules *SafetyRules + + initView uint64 // the current view at the beginning of the test case + endView uint64 // the expected current view at the end of the test case + parentProposal *model.Proposal + votingProposal *model.Proposal + qc *flow.QuorumCertificate + tc *flow.TimeoutCertificate + newview *model.NewViewEvent + ctx context.Context + stop context.CancelFunc } -func (f *Forks) MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) { - if f.qc == nil { - log.Fatal().Msgf("cannot make fork choice for curview: %v", curView) +func (es *EventHandlerSuite) SetupTest() { + finalized := uint64(3) + + es.parentProposal = createProposal(4, 3) + newestQC := createQC(es.parentProposal.Block) + + livenessData := &hotstuff.LivenessData{ + CurrentView: newestQC.View + 1, + NewestQC: newestQC, } - block, ok := f.blocks[f.qc.BlockID] - if !ok { - return nil, nil, fmt.Errorf("cannot block %V for fork choice qc", f.qc.BlockID) + es.ctx, es.stop = context.WithCancel(context.Background()) + + es.committee = NewCommittee(es.T()) + es.paceMaker = initPaceMaker(es.T(), es.ctx, livenessData) + es.forks = NewForks(es.T(), finalized) + es.persist = mocks.NewPersister(es.T()) + es.persist.On("PutStarted", mock.Anything).Return(nil).Maybe() + es.blockProducer = &BlockProducer{proposerID: es.committee.Self()} + es.safetyRules = NewSafetyRules(es.T()) + es.notifier = mocks.NewConsumer(es.T()) + es.notifier.On("OnEventProcessed").Maybe() + es.notifier.On("OnEnteringView", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnStart", mock.Anything).Maybe() + es.notifier.On("OnReceiveProposal", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnReceiveQc", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnReceiveTc", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnPartialTc", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnLocalTimeout", mock.Anything).Maybe() + es.notifier.On("OnCurrentViewDetails", mock.Anything, mock.Anything, mock.Anything).Maybe() + + eventhandler, err := NewEventHandler( + zerolog.New(os.Stderr), + es.paceMaker, + es.blockProducer, + es.forks, + es.persist, + es.committee, + es.safetyRules, + es.notifier) + require.NoError(es.T(), err) + + es.eventhandler = eventhandler + + es.initView = livenessData.CurrentView + es.endView = livenessData.CurrentView + // voting block is a block for the current view, which will trigger view change + es.votingProposal = createProposal(es.paceMaker.CurView(), es.parentProposal.Block.View) + es.qc = helper.MakeQC(helper.WithQCBlock(es.votingProposal.Block)) + + // create a TC that will trigger view change for current view, based on newest QC + es.tc = helper.MakeTC(helper.WithTCView(es.paceMaker.CurView()), + helper.WithTCNewestQC(es.votingProposal.Block.QC)) + es.newview = &model.NewViewEvent{ + View: es.votingProposal.Block.View + 1, // the vote for the voting proposals will trigger a view change to the next view } - log.Info().Msgf("forks.MakeForkChoice for view: %v, qc view: %v\n", curView, f.qc.View) - return f.qc, block, nil + + // add es.parentProposal into forks, otherwise we won't vote or propose based on it's QC sicne the parent is unknown + es.forks.proposals[es.parentProposal.Block.BlockID] = es.parentProposal } -// BlockProducer mock will always make a valid block -type BlockProducer struct{} +// TestStartNewView_ParentProposalNotFound tests next scenario: constructed TC, it contains NewestQC that references block that we +// don't know about, proposal can't be generated because we can't be sure that resulting block payload is valid. +func (es *EventHandlerSuite) TestStartNewView_ParentProposalNotFound() { + newestQC := helper.MakeQC(helper.WithQCView(es.initView + 10)) + tc := helper.MakeTC(helper.WithTCView(newestQC.View+1), + helper.WithTCNewestQC(newestQC)) -func (b *BlockProducer) MakeBlockProposal(qc *flow.QuorumCertificate, view uint64) (*model.Proposal, error) { - return createProposal(view, qc.View), nil + es.endView = tc.View + 1 + + // I'm leader for next block + es.committee.leaders[es.endView] = struct{}{} + + err := es.eventhandler.OnReceiveTc(tc) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "GetProposal", newestQC.BlockID) + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) } -// BlacklistValidator is Validator mock that consider all proposals are valid unless the proposal's BlockID exists -// in the invalidProposals key or unverifiable key -type BlacklistValidator struct { - mocks.Validator - invalidProposals map[flow.Identifier]struct{} - unverifiable map[flow.Identifier]struct{} - t require.TestingT +// TestOnReceiveProposal_StaleProposal test that proposals lower than finalized view are not processed at all +// we are not interested in this data because we already performed finalization of that height. +func (es *EventHandlerSuite) TestOnReceiveProposal_StaleProposal() { + proposal := createProposal(es.forks.FinalizedView()-1, es.forks.FinalizedView()-2) + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + es.forks.AssertNotCalled(es.T(), "AddBlock", proposal) } -func NewBlacklistValidator(t require.TestingT) *BlacklistValidator { - return &BlacklistValidator{ - invalidProposals: make(map[flow.Identifier]struct{}), - unverifiable: make(map[flow.Identifier]struct{}), - t: t, - } +// TestOnReceiveProposal_QCOlderThanCurView tests scenario: received a valid proposal with QC that has older view, +// the proposal's QC shouldn't trigger view change. +func (es *EventHandlerSuite) TestOnReceiveProposal_QCOlderThanCurView() { + proposal := createProposal(es.initView-1, es.initView-2) + + // should not trigger view change + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "AddProposal", proposal) } -func (v *BlacklistValidator) ValidateProposal(proposal *model.Proposal) error { - // check if is invalid - _, ok := v.invalidProposals[proposal.Block.BlockID] - if ok { - log.Info().Msgf("invalid proposal: %v\n", proposal.Block.View) - return model.InvalidBlockError{ - BlockID: proposal.Block.BlockID, - View: proposal.Block.View, - Err: fmt.Errorf("some error"), - } - } +// TestOnReceiveProposal_TCOlderThanCurView tests scenario: received a valid proposal with QC and TC that has older view, +// the proposal's QC shouldn't trigger view change. +func (es *EventHandlerSuite) TestOnReceiveProposal_TCOlderThanCurView() { + proposal := createProposal(es.initView-1, es.initView-3) + proposal.LastViewTC = helper.MakeTC(helper.WithTCView(proposal.Block.View-1), helper.WithTCNewestQC(proposal.Block.QC)) - // check if is unverifiable - _, ok = v.unverifiable[proposal.Block.BlockID] - if ok { - log.Info().Msgf("unverifiable proposal: %v\n", proposal.Block.View) - return model.ErrUnverifiableBlock - } + // should not trigger view change + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "AddProposal", proposal) +} - return nil +// TestOnReceiveProposal_NoVote tests scenario: received a valid proposal for cur view, but not a safe node to vote, and I'm the next leader +// should not vote. +func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote() { + proposal := createProposal(es.initView, es.initView-1) + + // I'm the next leader + es.committee.leaders[es.initView+1] = struct{}{} + // no vote for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "AddProposal", proposal) } -func TestEventHandler(t *testing.T) { - suite.Run(t, new(EventHandlerSuite)) +// TestOnReceiveProposal_NoVote_ParentProposalNotFound tests scenario: received a valid proposal for cur view, no parent for this proposal found +// should not vote. +func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote_ParentProposalNotFound() { + proposal := createProposal(es.initView, es.initView-1) + + // remove parent from known proposals + delete(es.forks.proposals, proposal.Block.QC.BlockID) + + // no vote for this proposal, no parent found + err := es.eventhandler.OnReceiveProposal(proposal) + require.Error(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "AddProposal", proposal) } -// EventHandlerSuite contains mocked state for testing event handler under different scenarios. -type EventHandlerSuite struct { - suite.Suite +// TestOnReceiveProposal_Vote_NextLeader tests scenario: received a valid proposal for cur view, safe to vote, I'm the next leader +// should vote and add vote to VoteAggregator. +func (es *EventHandlerSuite) TestOnReceiveProposal_Vote_NextLeader() { + proposal := createProposal(es.initView, es.initView-1) - eventhandler *EventHandler + // I'm the next leader + es.committee.leaders[es.initView+1] = struct{}{} + + // proposal is safe to vote + es.safetyRules.votable[proposal.Block.BlockID] = struct{}{} - paceMaker hotstuff.PaceMaker - forks *Forks - persist *mocks.Persister - blockProducer *BlockProducer - communicator *mocks.Communicator - committee *Committee - voteAggregator *mocks.VoteAggregator - voter *Voter - validator *BlacklistValidator - notifier hotstuff.Consumer + es.notifier.On("OnOwnVote", proposal.Block.BlockID, proposal.Block.View, mock.Anything, mock.Anything).Once() - initView uint64 - endView uint64 - votingBlock *model.Block - qc *flow.QuorumCertificate - newview *model.NewViewEvent + // vote should be created for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") } -func (es *EventHandlerSuite) SetupTest() { - finalized, curView := uint64(3), uint64(6) +// TestOnReceiveProposal_Vote_NotNextLeader tests scenario: received a valid proposal for cur view, safe to vote, I'm not the next leader +// should vote and send vote to next leader. +func (es *EventHandlerSuite) TestOnReceiveProposal_Vote_NotNextLeader() { + proposal := createProposal(es.initView, es.initView-1) - es.paceMaker = initPaceMaker(es.T(), curView) - es.forks = NewForks(es.T(), finalized) - es.persist = &mocks.Persister{} - es.persist.On("PutStarted", mock.Anything).Return(nil) - es.blockProducer = &BlockProducer{} - es.communicator = &mocks.Communicator{} - es.communicator.On("BroadcastProposalWithDelay", mock.Anything, mock.Anything).Return(nil) - es.communicator.On("SendVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) - es.committee = NewCommittee() - es.voteAggregator = &mocks.VoteAggregator{} - es.voter = NewVoter(es.T(), finalized) - es.validator = NewBlacklistValidator(es.T()) - es.notifier = ¬ifications.NoopConsumer{} + // proposal is safe to vote + es.safetyRules.votable[proposal.Block.BlockID] = struct{}{} - eventhandler, err := NewEventHandler( - zerolog.New(os.Stderr), - es.paceMaker, - es.blockProducer, - es.forks, - es.persist, - es.communicator, - es.committee, - es.voteAggregator, - es.voter, - es.validator, - es.notifier) + es.notifier.On("OnOwnVote", proposal.Block.BlockID, mock.Anything, mock.Anything, mock.Anything).Once() + + // vote should be created for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") +} - es.eventhandler = eventhandler +// TestOnReceiveProposal_ProposeAfterReceivingTC tests a scenario where we have received TC which advances to view where we are +// leader but no proposal can be created because we don't have parent proposal. After receiving missing parent proposal we have +// all available data to construct a valid proposal. We need to ensure this. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposeAfterReceivingQC() { - es.initView = curView - es.endView = curView - // voting block is a block for the current view, which will trigger view change - es.votingBlock = createBlockWithQC(es.paceMaker.CurView(), es.paceMaker.CurView()-1) - es.qc = &flow.QuorumCertificate{ - BlockID: es.votingBlock.BlockID, - View: es.votingBlock.View, - SignerIndices: nil, - SigData: nil, - } - es.newview = &model.NewViewEvent{ - View: es.votingBlock.View + 1, // the vote for the voting blocks will trigger a view change to the next view - } + qc := es.qc + + // first process QC this should advance view + err := es.eventhandler.OnReceiveQc(qc) + require.NoError(es.T(), err) + require.Equal(es.T(), qc.View+1, es.paceMaker.CurView(), "expect a view change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) + + // we are leader for current view + es.committee.leaders[es.paceMaker.CurView()] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + // it should broadcast a header as the same as current view + require.Equal(es.T(), es.paceMaker.CurView(), header.View) + }).Once() + + // processing this proposal shouldn't trigger view change since we have already seen QC. + // we have used QC to advance rounds, but no proposal was made because we were missing parent block + // when we have received parent block we can try proposing again. + err = es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + require.Equal(es.T(), qc.View+1, es.paceMaker.CurView(), "expect a view change") } -func (es *EventHandlerSuite) markInvalidProposal(blockID flow.Identifier) { - es.validator.invalidProposals[blockID] = struct{}{} +// TestOnReceiveProposal_ProposeAfterReceivingTC tests a scenario where we have received TC which advances to view where we are +// leader but no proposal can be created because we don't have parent proposal. After receiving missing parent proposal we have +// all available data to construct a valid proposal. We need to ensure this. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposeAfterReceivingTC() { + + // TC contains a QC.BlockID == es.votingProposal + tc := helper.MakeTC(helper.WithTCView(es.votingProposal.Block.View+1), + helper.WithTCNewestQC(es.qc)) + + // first process TC this should advance view + err := es.eventhandler.OnReceiveTc(tc) + require.NoError(es.T(), err) + require.Equal(es.T(), tc.View+1, es.paceMaker.CurView(), "expect a view change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) + + // we are leader for current view + es.committee.leaders[es.paceMaker.CurView()] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + // it should broadcast a header as the same as current view + require.Equal(es.T(), es.paceMaker.CurView(), header.View) + }).Once() + + // processing this proposal shouldn't trigger view change, since we have already seen QC. + // we have used QC to advance rounds, but no proposal was made because we were missing parent block + // when we have received parent block we can try proposing again. + err = es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + require.Equal(es.T(), tc.View+1, es.paceMaker.CurView(), "expect a view change") } -// a QC for current view triggered view change -func (es *EventHandlerSuite) TestQCBuiltViewChanged() { +// TestOnReceiveQc_HappyPath tests that building a QC for current view triggers view change. We are not leader for next +// round, so no proposal is expected. +func (es *EventHandlerSuite) TestOnReceiveQc_HappyPath() { // voting block exists - es.forks.blocks[es.votingBlock.BlockID] = es.votingBlock + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal // a qc is built - qc := createQC(es.votingBlock) + qc := createQC(es.votingProposal.Block) // new qc is added to forks // view changed @@ -405,364 +534,242 @@ func (es *EventHandlerSuite) TestQCBuiltViewChanged() { es.endView++ // not the leader of the newview // don't have block for the newview - // over - err := es.eventhandler.OnQCConstructed(qc) + err := es.eventhandler.OnReceiveQc(qc) require.NoError(es.T(), err, "if a vote can trigger a QC to be built,"+ "and the QC triggered a view change, then start new view") require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) } -// a QC for future view triggered view change -func (es *EventHandlerSuite) TestQCBuiltFutureViewChanged() { +// TestOnReceiveQc_FutureView tests that building a QC for future view triggers view change +func (es *EventHandlerSuite) TestOnReceiveQc_FutureView() { // voting block exists curView := es.paceMaker.CurView() // b1 is for current view // b2 and b3 is for future view, but branched out from the same parent as b1 - b1 := createBlockWithQC(curView, curView-1) - b2 := createBlockWithQC(curView+1, curView-1) - b3 := createBlockWithQC(curView+2, curView-1) + b1 := createProposal(curView, curView-1) + b2 := createProposal(curView+1, curView-1) + b3 := createProposal(curView+2, curView-1) // a qc is built // qc3 is for future view // qc2 is an older than qc3 // since vote aggregator can concurrently process votes and build qcs, // we prepare qcs at different view to be processed, and verify the view change. - qc1 := createQC(b1) - qc2 := createQC(b2) - qc3 := createQC(b3) + qc1 := createQC(b1.Block) + qc2 := createQC(b2.Block) + qc3 := createQC(b3.Block) - // all three blocks are known - es.forks.blocks[b1.BlockID] = b1 - es.forks.blocks[b2.BlockID] = b2 - es.forks.blocks[b3.BlockID] = b3 + // all three proposals are known + es.forks.proposals[b1.Block.BlockID] = b1 + es.forks.proposals[b2.Block.BlockID] = b2 + es.forks.proposals[b3.Block.BlockID] = b3 // test that qc for future view should trigger view change - err := es.eventhandler.OnQCConstructed(qc3) - endView := b3.View + 1 // next view + err := es.eventhandler.OnReceiveQc(qc3) + endView := b3.Block.View + 1 // next view require.NoError(es.T(), err, "if a vote can trigger a QC to be built,"+ "and the QC triggered a view change, then start new view") require.Equal(es.T(), endView, es.paceMaker.CurView(), "incorrect view change") // the same qc would not trigger view change - err = es.eventhandler.OnQCConstructed(qc3) - endView = b3.View + 1 // next view + err = es.eventhandler.OnReceiveQc(qc3) + endView = b3.Block.View + 1 // next view require.NoError(es.T(), err, "same qc should not trigger view change") require.Equal(es.T(), endView, es.paceMaker.CurView(), "incorrect view change") // old QCs won't trigger view change - err = es.eventhandler.OnQCConstructed(qc2) + err = es.eventhandler.OnReceiveQc(qc2) require.NoError(es.T(), err) require.Equal(es.T(), endView, es.paceMaker.CurView(), "incorrect view change") - err = es.eventhandler.OnQCConstructed(qc1) + err = es.eventhandler.OnReceiveQc(qc1) require.NoError(es.T(), err) require.Equal(es.T(), endView, es.paceMaker.CurView(), "incorrect view change") } -// in the newview, I'm not the leader, and I have the cur block, -// and the block is not a safe node, and I'm the next leader, and no qc built for this block. -func (es *EventHandlerSuite) TestInNewView_NotLeader_HasBlock_NoVote_IsNextLeader_NoQC() { - // voting block exists - es.forks.blocks[es.votingBlock.BlockID] = es.votingBlock - // a qc is built - qc := createQC(es.votingBlock) - // viewchanged +// TestOnReceiveQc_NextLeaderProposes tests that after receiving a valid proposal for cur view, and I'm the next leader, +// a QC can be built for the block, triggered view change, and I will propose +func (es *EventHandlerSuite) TestOnReceiveQc_NextLeaderProposes() { + proposal := createProposal(es.initView, es.initView-1) + qc := createQC(proposal.Block) + // I'm the next leader + es.committee.leaders[es.initView+1] = struct{}{} + // qc triggered view change es.endView++ - // not leader for newview + // I'm the leader of cur view (7) + // I'm not the leader of next view (8), trigger view change - // has block for newview - newviewblock := createBlockWithQC(es.newview.View, es.newview.View-1) - es.forks.blocks[newviewblock.BlockID] = newviewblock + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) - // I'm the next leader - es.committee.leaders[es.newview.View+1] = struct{}{} + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + // it should broadcast a header as the same as endView + require.Equal(es.T(), es.endView, header.View) + }).Once() - // no QC for the new view - err := es.eventhandler.OnQCConstructed(qc) + // after receiving proposal build QC and deliver it to event handler + err = es.eventhandler.OnReceiveQc(qc) require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.forks.AssertCalled(es.T(), "AddProposal", proposal) } -// TestInNewView_NotLeader_HasBlock_NoVote_IsNextLeader_QCBuilt_NoViewChange doesn't exist -// in the newview, I'm not the leader, and I have the cur block, -// and the block is not a safe node, and I'm the next leader, and a qc is built for this block, -// and the qc triggered view change. -func (es *EventHandlerSuite) TestInNewView_NotLeader_HasBlock_NoVote_IsNextLeader_QCBuilt_ViewChanged() { - // voting block exists - es.forks.blocks[es.votingBlock.BlockID] = es.votingBlock - // a qc is built - qc := createQC(es.votingBlock) - // viewchanged - es.endView++ - // not leader for newview - - // has block for newview - newviewblock := createBlockWithQC(es.newview.View, es.newview.View-1) - es.forks.blocks[newviewblock.BlockID] = newviewblock - - // not to vote for the new view block - +// TestOnReceiveQc_ProposeOnce tests that after constructing proposal we don't attempt to create another +// proposal for same view. +func (es *EventHandlerSuite) TestOnReceiveQc_ProposeOnce() { // I'm the next leader - es.committee.leaders[es.newview.View+1] = struct{}{} + es.committee.leaders[es.initView+1] = struct{}{} - // qc built for the new view block - nextQC := createQC(newviewblock) - // view change by this qc es.endView++ - err := es.eventhandler.OnQCConstructed(qc) - require.NoError(es.T(), err) - - // no broadcast shouldn't be made with first qc because we are not a leader - es.communicator.AssertNotCalled(es.T(), "BroadcastProposalWithDelay") + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Once() - err = es.eventhandler.OnQCConstructed(nextQC) + err := es.eventhandler.OnReceiveProposal(es.votingProposal) require.NoError(es.T(), err) - lastCall := es.communicator.Calls[len(es.communicator.Calls)-1] - // the last call is BroadcastProposal - require.Equal(es.T(), "BroadcastProposalWithDelay", lastCall.Method) - header, ok := lastCall.Arguments[0].(*flow.Header) - require.True(es.T(), ok) - // it should broadcast a header as the same as endView - require.Equal(es.T(), es.endView, header.View) -} - -// in the newview, I'm not the leader, and I have the cur block, -// and the block is a safe node to vote, and I'm the next leader, and no qc is built for this block. -func (es *EventHandlerSuite) TestInNewView_NotLeader_HasBlock_NotSafeNode_IsNextLeader_Voted_NoQC() { - // voting block exists - es.forks.blocks[es.votingBlock.BlockID] = es.votingBlock - // a qc is built - qc := createQC(es.votingBlock) - // viewchanged by new qc - es.endView++ - // not leader for newview - - // has block for newview - newviewblock := createBlockWithQC(es.newview.View, es.newview.View-1) - es.forks.blocks[newviewblock.BlockID] = newviewblock - - // not to vote for the new view block - - // I'm the next leader - es.committee.leaders[es.newview.View+1] = struct{}{} - - // no qc for the newview block + // constructing QC triggers making block proposal + err = es.eventhandler.OnReceiveQc(es.qc) + require.NoError(es.T(), err) - // should not trigger view change - err := es.eventhandler.OnQCConstructed(qc) + // receiving same proposal again triggers proposing logic + err = es.eventhandler.OnReceiveProposal(es.votingProposal) require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) } -// in the newview, I'm not the leader, and I have the cur block, -// and the block is not a safe node to vote, and I'm not the next leader -func (es *EventHandlerSuite) TestInNewView_NotLeader_HasBlock_NotSafeNode_NotNextLeader() { +// TestOnTCConstructed_HappyPath tests that building a TC for current view triggers view change +func (es *EventHandlerSuite) TestOnReceiveTc_HappyPath() { // voting block exists - es.forks.blocks[es.votingBlock.BlockID] = es.votingBlock - // a qc is built - qc := createQC(es.votingBlock) - // viewchanged by new qc - es.endView++ - - // view changed to newview - // I'm not the leader for newview + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal - // have received block for cur view - newviewblock := createBlockWithQC(es.newview.View, es.newview.View-1) - es.forks.blocks[newviewblock.BlockID] = newviewblock + // a tc is built + tc := helper.MakeTC(helper.WithTCView(es.initView), helper.WithTCNewestQC(es.votingProposal.Block.QC)) - // I'm not the next leader - // no vote for this block - // goes to the next view + // expect a view change es.endView++ - // not leader for next view - err := es.eventhandler.OnQCConstructed(qc) - require.NoError(es.T(), err, "if a vote can trigger a QC to be built,"+ - "and the QC triggered a view change, then start new view") + err := es.eventhandler.OnReceiveTc(tc) + require.NoError(es.T(), err, "TC should trigger a view change and start of new view") require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") } -// receiving an invalid proposal should not trigger view change -func (es *EventHandlerSuite) TestOnReceiveProposal_InvalidProposal_NoViewChange() { - proposal := createProposal(es.initView, es.initView-1) - // invalid proposal - es.markInvalidProposal(proposal.Block.BlockID) - es.voteAggregator.On("InvalidBlock", proposal).Return(nil).Once() +// TestOnTCConstructed_NextLeaderProposes tests that after receiving TC and advancing view we as next leader create a proposal +// and broadcast it +func (es *EventHandlerSuite) TestOnReceiveTc_NextLeaderProposes() { + es.committee.leaders[es.tc.View+1] = struct{}{} + es.endView++ - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) - require.Equal(es.T(), es.initView, es.paceMaker.CurView(), "incorrect view change") -} + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + // it should broadcast a header as the same as endView + require.Equal(es.T(), es.endView, header.View) -// received a valid proposal that has older view, and cannot build qc from votes for this block, -// the proposal's QC didn't trigger view change -func (es *EventHandlerSuite) TestOnReceiveProposal_OlderThanCurView_CannotBuildQCFromVotes_NoViewChange() { - proposal := createProposal(es.initView-1, es.initView-2) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() + // proposed block should contain valid newest QC and lastViewTC + expectedNewestQC := es.paceMaker.NewestQC() + proposal := model.ProposalFromFlow(header) + require.Equal(es.T(), expectedNewestQC, proposal.Block.QC) + require.Equal(es.T(), es.paceMaker.LastViewTC(), proposal.LastViewTC) + }).Once() - // can not build qc from votes for block - // should not trigger view change - err := es.eventhandler.OnReceiveProposal(proposal) + err := es.eventhandler.OnReceiveTc(es.tc) require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "TC didn't trigger view change") } -// received a valid proposal that has older view, and can built a qc from votes for this block, -// the proposal's QC didn't trigger view change -func (es *EventHandlerSuite) TestOnReceiveProposal_OlderThanCurView_CanBuildQCFromVotes_NoViewChange() { - proposal := createProposal(es.initView-1, es.initView-2) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() +// TestOnTimeout tests that event handler produces TimeoutObject and broadcasts it to other members of consensus +// committee. Additionally, It has to contribute TimeoutObject to timeout aggregation process by sending it to TimeoutAggregator. +func (es *EventHandlerSuite) TestOnTimeout() { + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(es.T(), ok) + // it should broadcast a TO with same view as endView + require.Equal(es.T(), es.endView, timeoutObject.View) + }).Once() - //should not trigger view change - err := es.eventhandler.OnReceiveProposal(proposal) + err := es.eventhandler.OnLocalTimeout() require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) -} - -// received a valid proposal that has newer view, and cannot build qc from votes for this block, -// the proposal's QC triggered view change -func (es *EventHandlerSuite) TestOnReceiveProposal_NewerThanCurView_CannotBuildQCFromVotes_ViewChange() { - proposal := createProposal(es.initView+1, es.initView) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - // can not build qc from votes for block - // block 7 triggered view change - es.endView++ - - // not leader of view 7, go to view 8 - es.endView++ - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) + // TimeoutObject shouldn't trigger view change require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) } -// received a valid proposal that has newer view, and can build qc from votes for this block, -// the proposal's QC triggered view change -func (es *EventHandlerSuite) TestOnReceiveProposal_NewerThanCurView_CanBuildQCFromVotes_ViewChange() { - proposal := createProposal(es.initView+1, es.initView) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() +// TestOnTimeout_SanityChecks tests a specific scenario where pacemaker have seen both QC and TC for previous view +// and EventHandler tries to produce a timeout object, such timeout object is invalid if both QC and TC is present, we +// need to make sure that EventHandler filters out TC for last view if we know about QC for same view. +func (es *EventHandlerSuite) TestOnTimeout_SanityChecks() { + // voting block exists + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal - es.forks.blocks[proposal.Block.BlockID] = proposal.Block + // a tc is built + tc := helper.MakeTC(helper.WithTCView(es.initView), helper.WithTCNewestQC(es.votingProposal.Block.QC)) - // trigged view change - es.endView++ - // the proposal is for next view, has block for next view, no vote, trigger view change + // expect a view change es.endView++ - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) -} - -// received a valid proposal whose QC that has newer view, and cannot build qc from votes for this block, -// the proposal's QC triggered view change -func (es *EventHandlerSuite) TestOnReceiveProposal_QCNewerThanCurView_CannotBuildQCFromVotes_ViewChanged() { - proposal := createProposal(es.initView+2, es.initView+1) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - - // can not build qc from votes for block - // block 8 triggered view change - es.endView = es.endView + 2 - - // not leader of view 8, go to view 9 - es.endView++ - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) + err := es.eventhandler.OnReceiveTc(tc) + require.NoError(es.T(), err, "TC should trigger a view change and start of new view") require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - require.Contains(es.T(), es.forks.blocks, proposal.Block.BlockID, "proposal block should be stored") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) -} -// received a valid proposal for cur view, but not a safe node to vote, and I'm the next leader, -// no qc for the block -func (es *EventHandlerSuite) TestOnReceiveProposal_ForCurView_NoVote_IsNextLeader_NoQC() { - proposal := createProposal(es.initView, es.initView-1) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - - // I'm the next leader - es.committee.leaders[es.initView+1] = struct{}{} - // no qc can be built for this block - err := es.eventhandler.OnReceiveProposal(proposal) + // receive a QC for the same view as the TC + qc := helper.MakeQC(helper.WithQCView(tc.View)) + err = es.eventhandler.OnReceiveQc(qc) require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) -} - -// received a valid proposal for cur view, but not a safe node to vote, and I'm the next leader, -// a qc can be built for the block, triggered view change -func (es *EventHandlerSuite) TestOnReceiveProposal_ForCurView_NoVote_IsNextLeader_QCBuilt_ViewChange() { - proposal := createProposal(es.initView, es.initView-1) - qc := createQC(proposal.Block) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - // I'm the next leader - es.committee.leaders[es.initView+1] = struct{}{} - // qc triggered view change - es.endView++ - // I'm the leader of cur view (7) - // I'm not the leader of next view (8), trigger view change + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "QC shouldn't trigger view change") + require.Equal(es.T(), tc, es.paceMaker.LastViewTC(), "invalid last view TC") + require.Equal(es.T(), qc, es.paceMaker.NewestQC(), "invalid newest QC") - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(es.T(), ok) + require.Equal(es.T(), es.endView, timeoutObject.View) + require.Equal(es.T(), qc, timeoutObject.NewestQC) + require.Nil(es.T(), timeoutObject.LastViewTC) + }).Once() - // after receiving proposal build QC and deliver it to event handler - err = es.eventhandler.OnQCConstructed(qc) + err = es.eventhandler.OnLocalTimeout() require.NoError(es.T(), err) - - lastCall := es.communicator.Calls[len(es.communicator.Calls)-1] - // the last call is BroadcastProposal - require.Equal(es.T(), "BroadcastProposalWithDelay", lastCall.Method) - header, ok := lastCall.Arguments[0].(*flow.Header) - require.True(es.T(), ok) - // it should broadcast a header as the same as endView - require.Equal(es.T(), es.endView, header.View) - - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) } -// received a unverifiable proposal for future view, no view change -func (es *EventHandlerSuite) TestOnReceiveProposal_Unverifiable() { - // qc.View is below the finalized view - proposal := createProposal(es.forks.finalized+2, es.forks.finalized-1) - - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - - // proposal is unverifiable - es.validator.unverifiable[proposal.Block.BlockID] = struct{}{} - - err := es.eventhandler.OnReceiveProposal(proposal) - require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - es.voteAggregator.AssertCalled(es.T(), "AddBlock", proposal) -} - -func (es *EventHandlerSuite) TestOnTimeout() { - err := es.eventhandler.OnLocalTimeout() - // timeout will trigger viewchange - es.endView++ - require.NoError(es.T(), err) - require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") +// TestOnTimeout_ReplicaEjected tests that EventHandler correctly handles possible errors from SafetyRules and doesn't broadcast +// timeout objects when replica is ejected. +func (es *EventHandlerSuite) TestOnTimeout_ReplicaEjected() { + es.Run("no-timeout", func() { + *es.safetyRules.SafetyRules = *mocks.NewSafetyRules(es.T()) + es.safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return(nil, model.NewNoTimeoutErrorf("")) + err := es.eventhandler.OnLocalTimeout() + require.NoError(es.T(), err, "should be handled as sentinel error") + }) + es.Run("create-timeout-exception", func() { + *es.safetyRules.SafetyRules = *mocks.NewSafetyRules(es.T()) + exception := errors.New("produce-timeout-exception") + es.safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return(nil, exception) + err := es.eventhandler.OnLocalTimeout() + require.ErrorIs(es.T(), err, exception, "expect a wrapped exception") + }) + es.notifier.AssertNotCalled(es.T(), "OnOwnTimeout", mock.Anything) } +// Test100Timeout tests that receiving 100 TCs for increasing views advances rounds func (es *EventHandlerSuite) Test100Timeout() { for i := 0; i < 100; i++ { - err := es.eventhandler.OnLocalTimeout() + tc := helper.MakeTC(helper.WithTCView(es.initView + uint64(i))) + err := es.eventhandler.OnReceiveTc(tc) es.endView++ require.NoError(es.T(), err) } require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") } -// a leader builds 100 blocks one after another +// TestLeaderBuild100Blocks tests scenario where leader builds 100 proposals one after another func (es *EventHandlerSuite) TestLeaderBuild100Blocks() { // I'm the leader for the first view es.committee.leaders[es.initView] = struct{}{} @@ -776,59 +783,257 @@ func (es *EventHandlerSuite) TestLeaderBuild100Blocks() { proposal := createProposal(es.initView+uint64(i), es.initView+uint64(i)-1) qc := createQC(proposal.Block) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - es.voteAggregator.On("AddVote", proposal.ProposerVote()).Return(nil).Once() + // for first proposal we need to store the parent otherwise it won't be voted for + if i == 0 { + parentBlock := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock(func(block *model.Block) { + block.BlockID = proposal.Block.QC.BlockID + block.View = proposal.Block.QC.View + }))) + es.forks.proposals[parentBlock.Block.BlockID] = parentBlock + } - es.voter.votable[proposal.Block.BlockID] = struct{}{} + es.safetyRules.votable[proposal.Block.BlockID] = struct{}{} // should trigger 100 view change es.endView++ + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + require.Equal(es.T(), proposal.Block.View+1, header.View) + }).Once() + es.notifier.On("OnOwnVote", proposal.Block.BlockID, proposal.Block.View, mock.Anything, mock.Anything).Once() + err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) - err = es.eventhandler.OnQCConstructed(qc) + err = es.eventhandler.OnReceiveQc(qc) require.NoError(es.T(), err) - - lastCall := es.communicator.Calls[len(es.communicator.Calls)-1] - require.Equal(es.T(), "BroadcastProposalWithDelay", lastCall.Method) - header, ok := lastCall.Arguments[0].(*flow.Header) - require.True(es.T(), ok) - require.Equal(es.T(), proposal.Block.View+1, header.View) } require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - require.Equal(es.T(), totalView, len(es.forks.blocks)) - es.voteAggregator.AssertExpectations(es.T()) + require.Equal(es.T(), totalView, (len(es.forks.proposals)-1)/2) } -// a follower receives 100 blocks +// TestFollowerFollows100Blocks tests scenario where follower receives 100 proposals one after another func (es *EventHandlerSuite) TestFollowerFollows100Blocks() { + // add parent proposal otherwise we can't propose + parentProposal := createProposal(es.initView, es.initView-1) + es.forks.proposals[parentProposal.Block.BlockID] = parentProposal for i := 0; i < 100; i++ { // create each proposal as if they are created by some leader - proposal := createProposal(es.initView+uint64(i), es.initView+uint64(i)-1) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - // as a follower, I receive these propsals + proposal := createProposal(es.initView+uint64(i)+1, es.initView+uint64(i)) + // as a follower, I receive these proposals err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) es.endView++ } require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - require.Equal(es.T(), 100, len(es.forks.blocks)) - es.voteAggregator.AssertExpectations(es.T()) + require.Equal(es.T(), 100, len(es.forks.proposals)-2) } -// a follower receives 100 forks built on top of the same block +// TestFollowerReceives100Forks tests scenario where follower receives 100 forks built on top of the same block func (es *EventHandlerSuite) TestFollowerReceives100Forks() { for i := 0; i < 100; i++ { // create each proposal as if they are created by some leader proposal := createProposal(es.initView+uint64(i)+1, es.initView-1) - es.voteAggregator.On("AddBlock", proposal).Return(nil).Once() - // as a follower, I receive these propsals + proposal.LastViewTC = helper.MakeTC(helper.WithTCView(es.initView+uint64(i)), + helper.WithTCNewestQC(proposal.Block.QC)) + // expect a view change since fork can be made only if last view has ended with TC. + es.endView++ + // as a follower, I receive these proposals err := es.eventhandler.OnReceiveProposal(proposal) require.NoError(es.T(), err) } require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") - require.Equal(es.T(), 100, len(es.forks.blocks)) - es.voteAggregator.AssertExpectations(es.T()) + require.Equal(es.T(), 100, len(es.forks.proposals)-1) +} + +// TestStart_PendingBlocksRecovery tests a scenario where node has unprocessed pending proposals that were not processed +// by event handler yet. After startup, we need to process all pending proposals. +func (es *EventHandlerSuite) TestStart_PendingBlocksRecovery() { + + var pendingProposals []*model.Proposal + proposal := createProposal(es.initView+1, es.initView) + pendingProposals = append(pendingProposals, proposal) + proposalWithTC := helper.MakeProposal(helper.WithBlock( + helper.MakeBlock( + helper.WithBlockView(es.initView+10), + helper.WithBlockQC(proposal.Block.QC))), + func(proposal *model.Proposal) { + proposal.LastViewTC = helper.MakeTC( + helper.WithTCView(proposal.Block.View-1), + helper.WithTCNewestQC(proposal.Block.QC)) + }, + ) + pendingProposals = append(pendingProposals, proposalWithTC) + proposal = createProposal(proposalWithTC.Block.View+1, proposalWithTC.Block.View) + pendingProposals = append(pendingProposals, proposal) + + for _, proposal := range pendingProposals { + es.forks.proposals[proposal.Block.BlockID] = proposal + } + + lastProposal := pendingProposals[len(pendingProposals)-1] + es.endView = lastProposal.Block.View + + es.forks.On("NewestView").Return(es.endView).Once() + + err := es.eventhandler.Start(es.ctx) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") +} + +// TestStart_ProposeOnce tests that after starting event handler we don't create proposal in case we have already proposed +// for this view. +func (es *EventHandlerSuite) TestStart_ProposeOnce() { + // I'm the next leader + es.committee.leaders[es.initView+1] = struct{}{} + + es.endView++ + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Once() + + err := es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + // constructing QC triggers making block proposal + err = es.eventhandler.OnReceiveQc(es.qc) + require.NoError(es.T(), err) + + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) + + es.forks.On("NewestView").Return(es.endView).Once() + + // Start triggers proposing logic, make sure that we don't propose again. + err = es.eventhandler.Start(es.ctx) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + // assert that broadcast wasn't trigger again + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) +} + +// TestCreateProposal_SanityChecks tests that proposing logic performs sanity checks when creating new block proposal. +// Specifically it tests a case where TC contains QC which: TC.View == TC.NewestQC.View +func (es *EventHandlerSuite) TestCreateProposal_SanityChecks() { + // round ended with TC where TC.View == TC.NewestQC.View + tc := helper.MakeTC(helper.WithTCView(es.initView), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCBlock(es.votingProposal.Block)))) + + es.forks.proposals[es.votingProposal.Block.BlockID] = es.votingProposal + + // I'm the next leader + es.committee.leaders[tc.View+1] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(es.T(), ok) + // we need to make sure that produced proposal contains only QC even if there is TC for previous view as well + require.Nil(es.T(), header.LastViewTC) + }).Once() + + err := es.eventhandler.OnReceiveTc(tc) + require.NoError(es.T(), err) + + require.Equal(es.T(), tc.NewestQC, es.paceMaker.NewestQC()) + require.Equal(es.T(), tc, es.paceMaker.LastViewTC()) + require.Equal(es.T(), tc.View+1, es.paceMaker.CurView(), "incorrect view change") +} + +// TestOnReceiveProposal_ProposalForActiveView tests that when receiving proposal for active we don't attempt to create a proposal +// Receiving proposal can trigger proposing logic only in case we have received missing block for past views. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposalForActiveView() { + // receive proposal where we are leader, meaning that we have produced this proposal + es.committee.leaders[es.votingProposal.Block.View] = struct{}{} + + err := es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) +} + +// TestOnPartialTcCreated_ProducedTimeout tests that when receiving partial TC for active view we will create a timeout object +// immediately. +func (es *EventHandlerSuite) TestOnPartialTcCreated_ProducedTimeout() { + partialTc := &hotstuff.PartialTcCreated{ + View: es.initView, + NewestQC: es.parentProposal.Block.QC, + LastViewTC: nil, + } + + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(es.T(), ok) + // it should broadcast a TO with same view as partialTc.View + require.Equal(es.T(), partialTc.View, timeoutObject.View) + }).Once() + + err := es.eventhandler.OnPartialTcCreated(partialTc) + require.NoError(es.T(), err) + + // partial TC shouldn't trigger view change + require.Equal(es.T(), partialTc.View, es.paceMaker.CurView(), "incorrect view change") +} + +// TestOnPartialTcCreated_NotActiveView tests that we don't create timeout object if partial TC was delivered for a past, non-current view. +// NOTE: it is not possible to receive a partial timeout for a FUTURE view, unless the partial timeout contains +// either a QC/TC allowing us to enter that view, therefore that case is not covered here. +// See TestOnPartialTcCreated_QcAndTcProcessing instead. +func (es *EventHandlerSuite) TestOnPartialTcCreated_NotActiveView() { + partialTc := &hotstuff.PartialTcCreated{ + View: es.initView - 1, + NewestQC: es.parentProposal.Block.QC, + } + + err := es.eventhandler.OnPartialTcCreated(partialTc) + require.NoError(es.T(), err) + + // partial TC shouldn't trigger view change + require.Equal(es.T(), es.initView, es.paceMaker.CurView(), "incorrect view change") + // we don't want to create timeout if partial TC was delivered for view different than active one. + es.notifier.AssertNotCalled(es.T(), "OnOwnTimeout", mock.Anything) +} + +// TestOnPartialTcCreated_QcAndTcProcessing tests that EventHandler processes QC and TC included in hotstuff.PartialTcCreated +// data structure. This tests cases like the following example: +// * the pacemaker is in view 10 +// * we observe a partial timeout for view 11 with a QC for view 10 +// * we should change to view 11 using the QC, then broadcast a timeout for view 11 +func (es *EventHandlerSuite) TestOnPartialTcCreated_QcAndTcProcessing() { + + testOnPartialTcCreated := func(partialTc *hotstuff.PartialTcCreated) { + es.endView++ + + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(es.T(), ok) + // it should broadcast a TO with same view as partialTc.View + require.Equal(es.T(), partialTc.View, timeoutObject.View) + }).Once() + + err := es.eventhandler.OnPartialTcCreated(partialTc) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endView, es.paceMaker.CurView(), "incorrect view change") + } + + es.Run("qc-triggered-view-change", func() { + partialTc := &hotstuff.PartialTcCreated{ + View: es.qc.View + 1, + NewestQC: es.qc, + } + testOnPartialTcCreated(partialTc) + }) + es.Run("tc-triggered-view-change", func() { + tc := helper.MakeTC(helper.WithTCView(es.endView), helper.WithTCNewestQC(es.qc)) + partialTc := &hotstuff.PartialTcCreated{ + View: tc.View + 1, + NewestQC: tc.NewestQC, + LastViewTC: tc, + } + testOnPartialTcCreated(partialTc) + }) } func createBlock(view uint64) *model.Block { @@ -839,7 +1044,7 @@ func createBlock(view uint64) *model.Block { }) return &model.Block{ BlockID: blockID, - View: uint64(view), + View: view, } } diff --git a/consensus/hotstuff/eventloop/event_loop.go b/consensus/hotstuff/eventloop/event_loop.go index fddc14e2a1c..95a06db8fda 100644 --- a/consensus/hotstuff/eventloop/event_loop.go +++ b/consensus/hotstuff/eventloop/event_loop.go @@ -9,6 +9,8 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/tracker" + "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" @@ -16,20 +18,28 @@ import ( "github.com/onflow/flow-go/module/metrics" ) -type proposalTask struct { - *model.Proposal - done chan struct{} +// queuedProposal is a helper structure that is used to transmit proposal in channel +// it contains an attached insertionTime that is used to measure how long we have waited between queening proposal and +// actually processing by `EventHandler`. +type queuedProposal struct { + proposal *model.Proposal + insertionTime time.Time } // EventLoop buffers all incoming events to the hotstuff EventHandler, and feeds EventHandler one event at a time. type EventLoop struct { *component.ComponentManager - log zerolog.Logger - eventHandler hotstuff.EventHandler - metrics module.HotstuffMetrics - proposals chan *proposalTask - quorumCertificates chan *flow.QuorumCertificate - startTime time.Time + log zerolog.Logger + eventHandler hotstuff.EventHandler + metrics module.HotstuffMetrics + proposals chan queuedProposal + newestSubmittedTc *tracker.NewestTCTracker + newestSubmittedQc *tracker.NewestQCTracker + newestSubmittedPartialTc *tracker.NewestPartialTcTracker + tcSubmittedNotifier engine.Notifier + qcSubmittedNotifier engine.Notifier + partialTcCreatedNotifier engine.Notifier + startTime time.Time } var _ hotstuff.EventLoop = (*EventLoop)(nil) @@ -37,16 +47,26 @@ var _ component.Component = (*EventLoop)(nil) // NewEventLoop creates an instance of EventLoop. func NewEventLoop(log zerolog.Logger, metrics module.HotstuffMetrics, eventHandler hotstuff.EventHandler, startTime time.Time) (*EventLoop, error) { - proposals := make(chan *proposalTask) - quorumCertificates := make(chan *flow.QuorumCertificate, 1) + // we will use a buffered channel to avoid blocking of caller + // we can't afford to drop messages since it undermines liveness, but we also want to avoid blocking of compliance + // engine. We assume that we should be able to process proposals faster than compliance engine feeds them, worst case + // we will fill the buffer and block compliance engine worker but that should happen only if compliance engine receives + // large number of blocks in short period of time(when catching up for instance). + // TODO(active-pacemaker) add metrics for length of inbound channels + proposals := make(chan queuedProposal, 1000) el := &EventLoop{ - log: log, - eventHandler: eventHandler, - metrics: metrics, - proposals: proposals, - quorumCertificates: quorumCertificates, - startTime: startTime, + log: log, + eventHandler: eventHandler, + metrics: metrics, + proposals: proposals, + tcSubmittedNotifier: engine.NewNotifier(), + qcSubmittedNotifier: engine.NewNotifier(), + partialTcCreatedNotifier: engine.NewNotifier(), + newestSubmittedTc: tracker.NewNewestTCTracker(), + newestSubmittedQc: tracker.NewNewestQCTracker(), + newestSubmittedPartialTc: tracker.NewNewestPartialTcTracker(), + startTime: startTime, } componentBuilder := component.NewComponentManagerBuilder() @@ -73,8 +93,7 @@ func NewEventLoop(log zerolog.Logger, metrics module.HotstuffMetrics, eventHandl } func (el *EventLoop) loop(ctx context.Context) error { - - err := el.eventHandler.Start() + err := el.eventHandler.Start(ctx) // must be called by the same go-routine that also executes the business logic! if err != nil { return fmt.Errorf("could not start event handler: %w", err) } @@ -86,8 +105,12 @@ func (el *EventLoop) loop(ctx context.Context) error { // if hotstuff hits any unknown error, it will exit the loop shutdownSignaled := ctx.Done() + timeoutCertificates := el.tcSubmittedNotifier.Channel() + quorumCertificates := el.qcSubmittedNotifier.Channel() + partialTCs := el.partialTcCreatedNotifier.Channel() + for { - // Giving timeout events the priority to be processed first + // Giving timeout events the priority to be processed first. // This is to prevent attacks from malicious nodes that attempt // to block honest nodes' pacemaker from progressing by sending // other events. @@ -100,15 +123,16 @@ func (el *EventLoop) loop(ctx context.Context) error { case <-shutdownSignaled: return nil - // if we receive a time out, process it and log errors + // processing timeout or partial TC event are top priority since + // they allow node to contribute to TC aggregation when replicas can't + // make progress on happy path case <-timeoutChannel: processStart := time.Now() - err := el.eventHandler.OnLocalTimeout() // measure how long it takes for a timeout event to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeTimeout) + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) if err != nil { return fmt.Errorf("could not process timeout: %w", err) @@ -120,6 +144,26 @@ func (el *EventLoop) loop(ctx context.Context) error { // Very important to start the for loop from the beginning, to continue the with the new timeout channel! continue + case <-partialTCs: + + processStart := time.Now() + err := el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) + + // measure how long it takes for a partial TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) + + if err != nil { + return fmt.Errorf("could no process partial created TC event: %w", err) + } + + // At this point, we have received and processed partial TC event, it could have resulted in several scenarios: + // 1. a view change with potential voting or proposal creation + // 2. a created and broadcast timeout object + // 3. QC and TC didn't result in view change and no timeout was created since we have already timed out or + // the partial TC was created for view different from current one. + + continue + default: // fall through to non-priority events } @@ -144,49 +188,49 @@ func (el *EventLoop) loop(ctx context.Context) error { err := el.eventHandler.OnLocalTimeout() // measure how long it takes for a timeout event to be processed - el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeTimeout) + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeLocalTimeout) if err != nil { return fmt.Errorf("could not process timeout: %w", err) } // if we have a new proposal, process it - case p := <-el.proposals: + case queuedItem := <-el.proposals: + // the wait duration is measured as how long it takes from a block being + // received to event handler commencing the processing of the block + el.metrics.HotStuffWaitDuration(time.Since(queuedItem.insertionTime), metrics.HotstuffEventTypeOnProposal) + // measure how long the event loop was idle waiting for an // incoming event el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - err := el.eventHandler.OnReceiveProposal(p.Proposal) - // done processing the proposal, notify the caller (usually the compliance engine) that - // this block has been processed. If the block is valid, protocol state should have it stored. - // useful when the caller is processing a range of blocks, and waiting for the current block - // to be processed by hotstuff before processing the next block. - close(p.done) + proposal := queuedItem.proposal + + err := el.eventHandler.OnReceiveProposal(proposal) // measure how long it takes for a proposal to be processed el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnProposal) if err != nil { - return fmt.Errorf("could not process proposal %v: %w", p.Block.BlockID, err) + return fmt.Errorf("could not process proposal %v: %w", proposal.Block.BlockID, err) } el.log.Info(). Dur("dur_ms", time.Since(processStart)). - Uint64("view", p.Block.View). - Hex("block_id", p.Block.BlockID[:]). + Uint64("view", proposal.Block.View). + Hex("block_id", proposal.Block.BlockID[:]). Msg("block proposal has been processed successfully") // if we have a new QC, process it - case qc := <-el.quorumCertificates: + case <-quorumCertificates: // measure how long the event loop was idle waiting for an // incoming event el.metrics.HotStuffIdleDuration(time.Since(idleStart)) processStart := time.Now() - - err := el.eventHandler.OnQCConstructed(qc) + err := el.eventHandler.OnReceiveQc(el.newestSubmittedQc.NewestQC()) // measure how long it takes for a QC to be processed el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnQC) @@ -194,42 +238,100 @@ func (el *EventLoop) loop(ctx context.Context) error { if err != nil { return fmt.Errorf("could not process QC: %w", err) } + + // if we have a new TC, process it + case <-timeoutCertificates: + // measure how long the event loop was idle waiting for an + // incoming event + el.metrics.HotStuffIdleDuration(time.Since(idleStart)) + + processStart := time.Now() + err := el.eventHandler.OnReceiveTc(el.newestSubmittedTc.NewestTC()) + + // measure how long it takes for a TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnTC) + + if err != nil { + return fmt.Errorf("could not process TC: %w", err) + } + + case <-partialTCs: + // measure how long the event loop was idle waiting for an + // incoming event + el.metrics.HotStuffIdleDuration(time.Since(idleStart)) + + processStart := time.Now() + err := el.eventHandler.OnPartialTcCreated(el.newestSubmittedPartialTc.NewestPartialTc()) + + // measure how long it takes for a partial TC to be processed + el.metrics.HotStuffBusyDuration(time.Since(processStart), metrics.HotstuffEventTypeOnPartialTc) + + if err != nil { + return fmt.Errorf("could no process partial created TC event: %w", err) + } } } } -// SubmitProposal pushes the received block to the blockheader channel -func (el *EventLoop) SubmitProposal(proposalHeader *flow.Header, parentView uint64) <-chan struct{} { - received := time.Now() - - proposal := &proposalTask{ - Proposal: model.ProposalFromFlow(proposalHeader, parentView), - done: make(chan struct{}), +// SubmitProposal pushes the received block to the proposals channel +func (el *EventLoop) SubmitProposal(proposal *model.Proposal) { + queueItem := queuedProposal{ + proposal: proposal, + insertionTime: time.Now(), } - select { - case el.proposals <- proposal: + case el.proposals <- queueItem: case <-el.ComponentManager.ShutdownSignal(): - return proposal.done + return } - // the wait duration is measured as how long it takes from a block being - // received to event handler commencing the processing of the block - el.metrics.HotStuffWaitDuration(time.Since(received), metrics.HotstuffEventTypeOnProposal) - return proposal.done } -// SubmitTrustedQC pushes the received QC to the quorumCertificates channel -func (el *EventLoop) SubmitTrustedQC(qc *flow.QuorumCertificate) { - received := time.Now() +// onTrustedQC pushes the received QC(which MUST be validated) to the quorumCertificates channel +func (el *EventLoop) onTrustedQC(qc *flow.QuorumCertificate) { + if el.newestSubmittedQc.Track(qc) { + el.qcSubmittedNotifier.Notify() + } +} - select { - case el.quorumCertificates <- qc: - case <-el.ComponentManager.ShutdownSignal(): - return +// onTrustedTC pushes the received TC(which MUST be validated) to the timeoutCertificates channel +func (el *EventLoop) onTrustedTC(tc *flow.TimeoutCertificate) { + if el.newestSubmittedTc.Track(tc) { + el.tcSubmittedNotifier.Notify() + } else if el.newestSubmittedQc.Track(tc.NewestQC) { + el.qcSubmittedNotifier.Notify() + } +} + +// OnTcConstructedFromTimeouts pushes the received TC to the timeoutCertificates channel +func (el *EventLoop) OnTcConstructedFromTimeouts(tc *flow.TimeoutCertificate) { + el.onTrustedTC(tc) +} + +// OnPartialTcCreated created a hotstuff.PartialTcCreated payload and pushes it into partialTcCreated buffered channel for +// further processing by EventHandler. Since we use buffered channel this function can block if buffer is full. +func (el *EventLoop) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + event := &hotstuff.PartialTcCreated{ + View: view, + NewestQC: newestQC, + LastViewTC: lastViewTC, } + if el.newestSubmittedPartialTc.Track(event) { + el.partialTcCreatedNotifier.Notify() + } +} + +// OnNewQcDiscovered pushes already validated QCs that were submitted from TimeoutAggregator to the event handler +func (el *EventLoop) OnNewQcDiscovered(qc *flow.QuorumCertificate) { + el.onTrustedQC(qc) +} + +// OnNewTcDiscovered pushes already validated TCs that were submitted from TimeoutAggregator to the event handler +func (el *EventLoop) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { + el.onTrustedTC(tc) +} - // the wait duration is measured as how long it takes from a qc being - // received to event handler commencing the processing of the qc - el.metrics.HotStuffWaitDuration(time.Since(received), metrics.HotstuffEventTypeOnQC) +// OnQcConstructedFromVotes implements hotstuff.QCCreatedConsumer and pushes received qc into processing pipeline. +func (el *EventLoop) OnQcConstructedFromVotes(qc *flow.QuorumCertificate) { + el.onTrustedQC(qc) } diff --git a/consensus/hotstuff/eventloop/event_loop_test.go b/consensus/hotstuff/eventloop/event_loop_test.go index 4bc7f2dba09..3f63b76f8d9 100644 --- a/consensus/hotstuff/eventloop/event_loop_test.go +++ b/consensus/hotstuff/eventloop/event_loop_test.go @@ -13,8 +13,11 @@ import ( "github.com/stretchr/testify/suite" "go.uber.org/atomic" + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" @@ -29,16 +32,16 @@ func TestEventLoop(t *testing.T) { type EventLoopTestSuite struct { suite.Suite - eh *mocks.EventHandlerV2 + eh *mocks.EventHandler cancel context.CancelFunc eventLoop *EventLoop } func (s *EventLoopTestSuite) SetupTest() { - s.eh = &mocks.EventHandlerV2{} - s.eh.On("Start").Return(nil).Maybe() - s.eh.On("TimeoutChannel").Return(time.NewTimer(10 * time.Second).C).Maybe() + s.eh = mocks.NewEventHandler(s.T()) + s.eh.On("Start", mock.Anything).Return(nil).Maybe() + s.eh.On("TimeoutChannel").Return(make(<-chan time.Time, 1)).Maybe() s.eh.On("OnLocalTimeout").Return(nil).Maybe() log := zerolog.New(io.Discard) @@ -72,36 +75,124 @@ func (s *EventLoopTestSuite) TestReadyDone() { // Test_SubmitQC tests that submitted proposal is eventually sent to event handler for processing func (s *EventLoopTestSuite) Test_SubmitProposal() { - proposal := unittest.BlockHeaderFixture() - expectedProposal := model.ProposalFromFlow(proposal, proposal.View-1) + proposal := helper.MakeProposal() processed := atomic.NewBool(false) - s.eh.On("OnReceiveProposal", expectedProposal).Run(func(args mock.Arguments) { + s.eh.On("OnReceiveProposal", proposal).Run(func(args mock.Arguments) { processed.Store(true) }).Return(nil).Once() - s.eventLoop.SubmitProposal(proposal, proposal.View-1) + s.eventLoop.SubmitProposal(proposal) require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) - s.eh.AssertExpectations(s.T()) } -// Test_SubmitQC tests that submitted QC is eventually sent to event handler for processing +// Test_SubmitQC tests that submitted QC is eventually sent to `EventHandler.OnReceiveQc` for processing func (s *EventLoopTestSuite) Test_SubmitQC() { - qc := unittest.QuorumCertificateFixture() + // qcIngestionFunction is the archetype for EventLoop.OnQcConstructedFromVotes and EventLoop.OnNewQcDiscovered + type qcIngestionFunction func(*flow.QuorumCertificate) + + testQCIngestionFunction := func(f qcIngestionFunction, qcView uint64) { + qc := helper.MakeQC(helper.WithQCView(qcView)) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveQc", qc).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(qc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + s.Run("QCs handed to EventLoop.OnQcConstructedFromVotes are forwarded to EventHandler", func() { + testQCIngestionFunction(s.eventLoop.OnQcConstructedFromVotes, 100) + }) + + s.Run("QCs handed to EventLoop.OnNewQcDiscovered are forwarded to EventHandler", func() { + testQCIngestionFunction(s.eventLoop.OnNewQcDiscovered, 101) + }) +} + +// Test_SubmitTC tests that submitted TC is eventually sent to `EventHandler.OnReceiveTc` for processing +func (s *EventLoopTestSuite) Test_SubmitTC() { + // tcIngestionFunction is the archetype for EventLoop.OnTcConstructedFromTimeouts and EventLoop.OnNewTcDiscovered + type tcIngestionFunction func(*flow.TimeoutCertificate) + + testTCIngestionFunction := func(f tcIngestionFunction, tcView uint64) { + tc := helper.MakeTC(helper.WithTCView(tcView)) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveTc", tc).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(tc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + s.Run("TCs handed to EventLoop.OnTcConstructedFromTimeouts are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnTcConstructedFromTimeouts, 100) + }) + + s.Run("TCs handed to EventLoop.OnNewTcDiscovered are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnNewTcDiscovered, 101) + }) +} + +// Test_SubmitTC_IngestNewestQC tests that included QC in TC is eventually sent to `EventHandler.OnReceiveQc` for processing +func (s *EventLoopTestSuite) Test_SubmitTC_IngestNewestQC() { + // tcIngestionFunction is the archetype for EventLoop.OnTcConstructedFromTimeouts and EventLoop.OnNewTcDiscovered + type tcIngestionFunction func(*flow.TimeoutCertificate) + + testTCIngestionFunction := func(f tcIngestionFunction, tcView, qcView uint64) { + tc := helper.MakeTC(helper.WithTCView(tcView), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(qcView)))) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveQc", tc.NewestQC).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(tc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + // process initial TC, this will track the newest TC + s.eh.On("OnReceiveTc", mock.Anything).Return(nil).Once() + s.eventLoop.OnTcConstructedFromTimeouts(helper.MakeTC( + helper.WithTCView(100), + helper.WithTCNewestQC( + helper.MakeQC( + helper.WithQCView(80), + ), + ), + )) + + s.Run("QCs handed to EventLoop.OnTcConstructedFromTimeouts are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnTcConstructedFromTimeouts, 100, 99) + }) + + s.Run("QCs handed to EventLoop.OnNewTcDiscovered are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnNewTcDiscovered, 100, 100) + }) +} + +// Test_OnPartialTcCreated tests that event loop delivers partialTcCreated events to event handler. +func (s *EventLoopTestSuite) Test_OnPartialTcCreated() { + view := uint64(1000) + newestQC := helper.MakeQC(helper.WithQCView(view - 10)) + lastViewTC := helper.MakeTC(helper.WithTCView(view-1), helper.WithTCNewestQC(newestQC)) + processed := atomic.NewBool(false) - s.eh.On("OnQCConstructed", qc).Run(func(args mock.Arguments) { + partialTcCreated := &hotstuff.PartialTcCreated{ + View: view, + NewestQC: newestQC, + LastViewTC: lastViewTC, + } + s.eh.On("OnPartialTcCreated", partialTcCreated).Run(func(args mock.Arguments) { processed.Store(true) }).Return(nil).Once() - s.eventLoop.SubmitTrustedQC(qc) + s.eventLoop.OnPartialTcCreated(view, newestQC, lastViewTC) require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) - s.eh.AssertExpectations(s.T()) } // TestEventLoop_Timeout tests that event loop delivers timeout events to event handler under pressure func TestEventLoop_Timeout(t *testing.T) { - eh := &mocks.EventHandlerV2{} + eh := &mocks.EventHandler{} processed := atomic.NewBool(false) - eh.On("Start").Return(nil).Once() - eh.On("TimeoutChannel").Return(time.NewTimer(100 * time.Millisecond).C) - eh.On("OnQCConstructed", mock.Anything).Return(nil).Maybe() + eh.On("Start", mock.Anything).Return(nil).Once() + eh.On("OnReceiveQc", mock.Anything).Return(nil).Maybe() eh.On("OnReceiveProposal", mock.Anything).Return(nil).Maybe() eh.On("OnLocalTimeout").Run(func(args mock.Arguments) { processed.Store(true) @@ -112,6 +203,8 @@ func TestEventLoop_Timeout(t *testing.T) { eventLoop, err := NewEventLoop(log, metrics.NewNoopCollector(), eh, time.Time{}) require.NoError(t, err) + eh.On("TimeoutChannel").Return(time.After(100 * time.Millisecond)) + ctx, cancel := context.WithCancel(context.Background()) signalerCtx, _ := irrecoverable.WithSignaler(ctx) eventLoop.Start(signalerCtx) @@ -128,20 +221,19 @@ func TestEventLoop_Timeout(t *testing.T) { defer wg.Done() for !processed.Load() { qc := unittest.QuorumCertificateFixture() - eventLoop.SubmitTrustedQC(qc) + eventLoop.OnQcConstructedFromVotes(qc) } }() go func() { defer wg.Done() for !processed.Load() { - proposal := unittest.BlockHeaderFixture() - eventLoop.SubmitProposal(proposal, proposal.View-1) + eventLoop.SubmitProposal(helper.MakeProposal()) } }() require.Eventually(t, processed.Load, time.Millisecond*200, time.Millisecond*10) - wg.Wait() + unittest.AssertReturnsBefore(t, func() { wg.Wait() }, time.Millisecond*200) cancel() unittest.RequireCloseBefore(t, eventLoop.Done(), 100*time.Millisecond, "event loop not stopped") @@ -150,9 +242,9 @@ func TestEventLoop_Timeout(t *testing.T) { // TestReadyDoneWithStartTime tests that event loop correctly starts and schedules start of processing // when startTime argument is used func TestReadyDoneWithStartTime(t *testing.T) { - eh := &mocks.EventHandlerV2{} - eh.On("Start").Return(nil) - eh.On("TimeoutChannel").Return(time.NewTimer(10 * time.Second).C) + eh := &mocks.EventHandler{} + eh.On("Start", mock.Anything).Return(nil) + eh.On("TimeoutChannel").Return(make(<-chan time.Time, 1)) eh.On("OnLocalTimeout").Return(nil) metrics := metrics.NewNoopCollector() @@ -178,7 +270,7 @@ func TestReadyDoneWithStartTime(t *testing.T) { parentBlock := unittest.BlockHeaderFixture() block := unittest.BlockHeaderWithParentFixture(parentBlock) - eventLoop.SubmitProposal(block, parentBlock.View) + eventLoop.SubmitProposal(model.ProposalFromFlow(block)) unittest.RequireCloseBefore(t, done, startTimeDuration+100*time.Millisecond, "proposal wasn't received") cancel() diff --git a/consensus/hotstuff/follower/follower.go b/consensus/hotstuff/follower/follower.go index 75bd137af12..cef8b3d0c1b 100644 --- a/consensus/hotstuff/follower/follower.go +++ b/consensus/hotstuff/follower/follower.go @@ -7,7 +7,6 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/utils/logging" ) @@ -21,14 +20,14 @@ import ( type FollowerLogic struct { log zerolog.Logger validator hotstuff.Validator - finalizationLogic forks.Finalizer + finalizationLogic hotstuff.Forks } // New creates a new FollowerLogic instance func New( log zerolog.Logger, validator hotstuff.Validator, - finalizationLogic forks.Finalizer, + finalizationLogic hotstuff.Forks, ) (*FollowerLogic, error) { return &FollowerLogic{ log: log.With().Str("hotstuff", "follower").Logger(), @@ -46,32 +45,35 @@ func (f *FollowerLogic) FinalizedBlock() *model.Block { func (f *FollowerLogic) AddBlock(blockProposal *model.Proposal) error { // validate the block. skip if the proposal is invalid err := f.validator.ValidateProposal(blockProposal) - if model.IsInvalidBlockError(err) { - f.log.Warn().Err(err).Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Msg("invalid proposal") - return nil - } - if errors.Is(err, model.ErrUnverifiableBlock) { - f.log.Warn(). - Hex("block_id", logging.ID(blockProposal.Block.BlockID)). - Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). - Msg("unverifiable proposal") - - // even if the block is unverifiable because the QC has been - // pruned, it still needs to be added to the forks, otherwise, - // a new block with a QC to this block will fail to be added - // to forks and crash the event loop. - } else if err != nil { - return fmt.Errorf("cannot validate block proposal %x: %w", blockProposal.Block.BlockID, err) + if err != nil { + if model.IsInvalidBlockError(err) { + f.log.Warn().Err(err). + Hex("block_id", logging.ID(blockProposal.Block.BlockID)). + Msg("invalid proposal") + return nil + } else if errors.Is(err, model.ErrViewForUnknownEpoch) { + f.log.Warn().Err(err). + Hex("block_id", logging.ID(blockProposal.Block.BlockID)). + Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). + Uint64("block_view", blockProposal.Block.View). + Msg("proposal for unknown epoch") + return nil + } else if errors.Is(err, model.ErrUnverifiableBlock) { + f.log.Warn().Err(err). + Hex("block_id", logging.ID(blockProposal.Block.BlockID)). + Hex("qc_block_id", logging.ID(blockProposal.Block.QC.BlockID)). + Uint64("block_view", blockProposal.Block.View). + Msg("unverifiable proposal") + // even if the block is unverifiable because the QC has been + // pruned, it still needs to be added to the forks, otherwise, + // a new block with a QC to this block will fail to be added + // to forks and crash the event loop. + } else if err != nil { + return fmt.Errorf("cannot validate block proposal %x: %w", blockProposal.Block.BlockID, err) + } } - // as a sanity check, we run the finalization logic's internal validation on the block - if err := f.finalizationLogic.VerifyBlock(blockProposal.Block); err != nil { - // this should never happen: the block was found to be valid by the validator - // if the finalization logic's internal validation errors, we have a bug - return fmt.Errorf("invaid block passed validation: %w", err) - } - err = f.finalizationLogic.AddBlock(blockProposal.Block) + err = f.finalizationLogic.AddProposal(blockProposal) if err != nil { return fmt.Errorf("finalization logic cannot process block proposal %x: %w", blockProposal.Block.BlockID, err) } diff --git a/consensus/hotstuff/follower_loop.go b/consensus/hotstuff/follower_loop.go index 3bddc925f8a..ae9289c1860 100644 --- a/consensus/hotstuff/follower_loop.go +++ b/consensus/hotstuff/follower_loop.go @@ -6,34 +6,43 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/consensus/hotstuff/runner" - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/utils/logging" ) -// proposalTask struct used to send a proposal and done channel in one message -type proposalTask struct { - *model.Proposal - done chan struct{} // closed when the proposal has finished being processed -} - -// FollowerLoop implements interface FollowerLoop +// FollowerLoop implements interface module.HotStuffFollower. +// FollowerLoop buffers all incoming events to the hotstuff FollowerLogic, and feeds FollowerLogic one event at a time +// using a worker thread. +// Concurrency safe. type FollowerLoop struct { + *component.ComponentManager log zerolog.Logger followerLogic FollowerLogic - proposals chan *proposalTask - - runner runner.SingleRunner // lock for preventing concurrent state transitions + proposals chan *model.Proposal } +var _ component.Component = (*FollowerLoop)(nil) +var _ module.HotStuffFollower = (*FollowerLoop)(nil) + // NewFollowerLoop creates an instance of EventLoop func NewFollowerLoop(log zerolog.Logger, followerLogic FollowerLogic) (*FollowerLoop, error) { - return &FollowerLoop{ + // TODO(active-pacemaker) add metrics for length of inbound channels + // we will use a buffered channel to avoid blocking of caller + proposals := make(chan *model.Proposal, 1000) + + fl := &FollowerLoop{ log: log, followerLogic: followerLogic, - proposals: make(chan *proposalTask), - runner: runner.NewSingleRunner(), - }, nil + proposals: proposals, + } + + fl.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(fl.loop). + Build() + + return fl, nil } // SubmitProposal feeds a new block proposal (header) into the FollowerLoop. @@ -41,14 +50,14 @@ func NewFollowerLoop(log zerolog.Logger, followerLogic FollowerLogic) (*Follower // // Block proposals must be submitted in order, i.e. a proposal's parent must // have been previously processed by the FollowerLoop. -func (fl *FollowerLoop) SubmitProposal(proposalHeader *flow.Header, parentView uint64) <-chan struct{} { +func (fl *FollowerLoop) SubmitProposal(proposal *model.Proposal) { received := time.Now() - proposal := &proposalTask{ - Proposal: model.ProposalFromFlow(proposalHeader, parentView), - done: make(chan struct{}), - } - fl.proposals <- proposal + select { + case fl.proposals <- proposal: + case <-fl.ComponentManager.ShutdownSignal(): + return + } // the busy duration is measured as how long it takes from a block being // received to a block being handled by the event handler. @@ -57,16 +66,15 @@ func (fl *FollowerLoop) SubmitProposal(proposalHeader *flow.Header, parentView u Uint64("view", proposal.Block.View). Dur("busy_duration", busyDuration). Msg("busy duration to handle a proposal") - - return proposal.done } -// loop will synchronously processes all events. +// loop will synchronously process all events. // All errors from FollowerLogic are fatal: // - known critical error: some prerequisites of the HotStuff follower have been broken // - unknown critical error: bug-related -func (fl *FollowerLoop) loop() { - shutdownSignal := fl.runner.ShutdownSignal() +func (fl *FollowerLoop) loop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + shutdownSignal := fl.ComponentManager.ShutdownSignal() for { select { // to ensure we are not skipping over a termination signal case <-shutdownSignal: @@ -76,31 +84,17 @@ func (fl *FollowerLoop) loop() { select { case p := <-fl.proposals: - err := fl.followerLogic.AddBlock(p.Proposal) - close(p.done) - + err := fl.followerLogic.AddBlock(p) if err != nil { // all errors are fatal fl.log.Error(). Hex("block_id", logging.ID(p.Block.BlockID)). Uint64("view", p.Block.View). Err(err). - Msg("terminating FollowerLoop") - return + Msg("irrecoverable follower loop error") + ctx.Throw(err) } case <-shutdownSignal: return } } } - -// Ready implements interface module.ReadyDoneAware -// Method call will starts the FollowerLoop's internal processing loop. -// Multiple calls are handled gracefully and the follower will only start once. -func (fl *FollowerLoop) Ready() <-chan struct{} { - return fl.runner.Start(fl.loop) -} - -// Done implements interface module.ReadyDoneAware -func (fl *FollowerLoop) Done() <-chan struct{} { - return fl.runner.Abort() -} diff --git a/consensus/hotstuff/forks.go b/consensus/hotstuff/forks.go index 418f51b0cb2..8cdbdc241d2 100644 --- a/consensus/hotstuff/forks.go +++ b/consensus/hotstuff/forks.go @@ -5,8 +5,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// Forks encapsulated Finalization Logic and ForkChoice rule in one component. -// Forks maintains an in-memory data-structure of all blocks whose view-number is larger or equal to +// Forks maintains an in-memory data-structure of all proposals whose view-number is larger or equal to // the latest finalized block. The latest finalized block is defined as the finalized block with the largest view number. // When adding blocks, Forks automatically updates its internal state (including finalized blocks). // Furthermore, blocks whose view number is smaller than the latest finalized block are pruned automatically. @@ -16,65 +15,30 @@ import ( // (without missing interim ancestors). If this condition is violated, Forks will raise an error // and ignore the block. type Forks interface { - ForksReader - // AddBlock adds the block to Forks. This might cause an update of the finalized block - // and pruning of older blocks. - // Handles duplicated addition of blocks (at the potential cost of additional computation time). - // PREREQUISITE: - // Forks must be able to connect `block` to its latest finalized block - // (without missing interim ancestors). Otherwise, an error is raised. - // When the new block causes the conflicting finalized blocks, it will return - // Might error with ByzantineThresholdExceededError (e.g. if finalizing conflicting forks) - AddBlock(block *model.Block) error - - // AddQC adds a quorum certificate to Forks. - // Will error in case the block referenced by the qc is unknown. - // Might error with ByzantineThresholdExceededError (e.g. if two conflicting QCs for the - // same view are found) - AddQC(qc *flow.QuorumCertificate) error - - // MakeForkChoice prompts the ForkChoice to generate a fork choice for the - // current view `curView`. The fork choice is a qc that should be used for - // building the primaries block. - // It returns a qc and the block that the qc is pointing to. - // - // PREREQUISITE: - // ForkChoice cannot generate ForkChoices retroactively for past views. - // If used correctly, MakeForkChoice should only ever have processed QCs - // whose view is smaller than curView, for the following reason: - // Processing a QC with view v should result in the PaceMaker being in - // view v+1 or larger. Hence, given that the current View is curView, - // all QCs should have view < curView. - // To prevent accidental misusage, ForkChoices will error if `curView` - // is smaller than the view of any qc ForkChoice has seen. - // Note that tracking the view of the newest qc is for safety purposes - // and _independent_ of the fork-choice rule. - MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) -} - -// ForksReader only reads the forks' state -type ForksReader interface { + // GetProposalsForView returns all BlockProposals at the given view number. + GetProposalsForView(view uint64) []*model.Proposal - // IsSafeBlock returns true if block is safe to vote for - // (according to the definition in https://arxiv.org/abs/1803.05069v6). - // - // In the current architecture, the block is stored _before_ evaluating its safety. - // Consequently, IsSafeBlock accepts only known, valid blocks. Should a block be - // unknown (not previously added to Forks) or violate some consistency requirements, - // IsSafeBlock errors. All errors are fatal. - IsSafeBlock(block *model.Block) bool - - // GetBlocksForView returns all BlockProposals at the given view number. - GetBlocksForView(view uint64) []*model.Block - - // GetBlock returns (BlockProposal, true) if the block with the specified + // GetProposal returns (BlockProposal, true) if the block with the specified // id was found (nil, false) otherwise. - GetBlock(id flow.Identifier) (*model.Block, bool) + GetProposal(id flow.Identifier) (*model.Proposal, bool) // FinalizedView returns the largest view number where a finalized block is known FinalizedView() uint64 // FinalizedBlock returns the finalized block with the largest view number FinalizedBlock() *model.Block + + // NewestView returns the largest view number of all proposals that were added to Forks. + NewestView() uint64 + + // AddProposal adds the block proposal to Forks. This might cause an update of the finalized block + // and pruning of older blocks. + // Handles duplicated addition of blocks (at the potential cost of additional computation time). + // PREREQUISITE: + // Forks must be able to connect `proposal` to its latest finalized block + // (without missing interim ancestors). Otherwise, an exception is raised. + // Expected errors during normal operations: + // * model.ByzantineThresholdExceededError - new block results in conflicting finalized blocks + AddProposal(proposal *model.Proposal) error } diff --git a/consensus/hotstuff/forks/test/block_builder.go b/consensus/hotstuff/forks/block_builder_test.go similarity index 53% rename from consensus/hotstuff/forks/test/block_builder.go rename to consensus/hotstuff/forks/block_builder_test.go index 7f9d7488d22..3624817672b 100644 --- a/consensus/hotstuff/forks/test/block_builder.go +++ b/consensus/hotstuff/forks/block_builder_test.go @@ -1,34 +1,37 @@ -package test +package forks import ( "fmt" - "github.com/onflow/flow-go/consensus/hotstuff/forks" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) // BlockView specifies the data to create a block type BlockView struct { - View uint64 // the view of the block to be created - - // the version of the block for that view. useful for creating a different block of the same view with a different version + // View is the view of the block to be created + View uint64 + // BlockVersion is the version of the block for that view. + // Useful for creating conflicting blocks at the same view. BlockVersion int - - QCView uint64 // the view for the QC to be built on top of - - // the version of the QC for that view. + // QCView is the view of the QC embedded in this block (also: the view of the block's parent) + QCView uint64 + // QCVersion is the version of the QC for that view. QCVersion int } +// QCIndex returns a unique identifier for the block's QC. func (bv *BlockView) QCIndex() string { return fmt.Sprintf("%v-%v", bv.QCView, bv.QCVersion) } +// BlockIndex returns a unique identifier for the block. func (bv *BlockView) BlockIndex() string { return fmt.Sprintf("%v-%v", bv.View, bv.BlockVersion) } +// BlockBuilder is a test utility for creating block structure fixtures. type BlockBuilder struct { blockViews []*BlockView } @@ -39,6 +42,7 @@ func NewBlockBuilder() *BlockBuilder { } } +// Add adds a block with the given qcView and blockView. func (f *BlockBuilder) Add(qcView uint64, blockView uint64) { f.blockViews = append(f.blockViews, &BlockView{ View: blockView, @@ -46,21 +50,32 @@ func (f *BlockBuilder) Add(qcView uint64, blockView uint64) { }) } -// [3,4] denotes a block of view 4, with a qc of view 3. -// [3,4'] denotes a block of view 4, with a qc of view 3, but has a different BlockID than [3,4], +// GenesisBlock returns the genesis block, which is always finalized. +func (f *BlockBuilder) GenesisBlock() *model.Block { + return makeGenesis().Block +} + +// AddVersioned adds a block with the given qcView and blockView. +// In addition the version identifier of the QC embedded within the block +// is specified by `qcVersion`. The version identifier for the block itself +// (primarily for emulating different payloads) is specified by `blockVersion`. +// [3,4] denotes a block of view 4, with a qc of view 3 +// [3,4'] denotes a block of view 4, with a qc of view 3, but has a different BlockID than [3,4] // [3,4'] can be created by AddVersioned(3, 4, 0, 1) // [3',4] can be created by AddVersioned(3, 4, 1, 0) -func (f *BlockBuilder) AddVersioned(qcView uint64, blockView uint64, qcversion int, blockversion int) { +func (f *BlockBuilder) AddVersioned(qcView uint64, blockView uint64, qcVersion int, blockVersion int) { f.blockViews = append(f.blockViews, &BlockView{ View: blockView, QCView: qcView, - BlockVersion: blockversion, - QCVersion: qcversion, + BlockVersion: blockVersion, + QCVersion: qcVersion, }) } -func (f *BlockBuilder) Blocks() ([]*model.Block, error) { - blocks := make([]*model.Block, 0, len(f.blockViews)) +// Blocks returns a list of all blocks added to the BlockBuilder. +// Returns an error if the blocks do not form a connected tree rooted at genesis. +func (f *BlockBuilder) Blocks() ([]*model.Proposal, error) { + blocks := make([]*model.Proposal, 0, len(f.blockViews)) genesisBQ := makeGenesis() genesisBV := &BlockView{ @@ -77,19 +92,27 @@ func (f *BlockBuilder) Blocks() ([]*model.Block, error) { return nil, fmt.Errorf("test fail: no qc found for qc index: %v", bv.QCIndex()) } payloadHash := makePayloadHash(bv.View, qc, bv.BlockVersion) - block := &model.Block{ - View: bv.View, - QC: qc, - PayloadHash: payloadHash, + var lastViewTC *flow.TimeoutCertificate + if qc.View+1 != bv.View { + lastViewTC = helper.MakeTC(helper.WithTCView(bv.View - 1)) + } + proposal := &model.Proposal{ + Block: &model.Block{ + View: bv.View, + QC: qc, + PayloadHash: payloadHash, + }, + LastViewTC: lastViewTC, + SigData: nil, } - block.BlockID = makeBlockID(block) + proposal.Block.BlockID = makeBlockID(proposal.Block) - blocks = append(blocks, block) + blocks = append(blocks, proposal) - // generate QC for the new block + // generate QC for the new proposal qcs[bv.BlockIndex()] = &flow.QuorumCertificate{ - View: block.View, - BlockID: block.BlockID, + View: proposal.Block.View, + BlockID: proposal.Block.BlockID, SignerIndices: nil, SigData: nil, } @@ -122,7 +145,7 @@ func makeBlockID(block *model.Block) flow.Identifier { }) } -func makeGenesis() *forks.BlockQC { +func makeGenesis() *BlockQC { genesis := &model.Block{ View: 1, } @@ -132,7 +155,7 @@ func makeGenesis() *forks.BlockQC { View: 1, BlockID: genesis.BlockID, } - genesisBQ := &forks.BlockQC{ + genesisBQ := &BlockQC{ Block: genesis, QC: genesisQC, } diff --git a/consensus/hotstuff/forks/blockcontainer.go b/consensus/hotstuff/forks/blockcontainer.go new file mode 100644 index 00000000000..2681f5d57c6 --- /dev/null +++ b/consensus/hotstuff/forks/blockcontainer.go @@ -0,0 +1,23 @@ +package forks + +import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/forest" +) + +// BlockContainer wraps a block proposal to implement forest.Vertex +// so the proposal can be stored in forest.LevelledForest +type BlockContainer struct { + Proposal *model.Proposal +} + +var _ forest.Vertex = (*BlockContainer)(nil) + +// Functions implementing forest.Vertex + +func (b *BlockContainer) VertexID() flow.Identifier { return b.Proposal.Block.BlockID } +func (b *BlockContainer) Level() uint64 { return b.Proposal.Block.View } +func (b *BlockContainer) Parent() (flow.Identifier, uint64) { + return b.Proposal.Block.QC.BlockID, b.Proposal.Block.QC.View +} diff --git a/consensus/hotstuff/forks/finalizer.go b/consensus/hotstuff/forks/finalizer.go deleted file mode 100644 index 9a6f5e8d87b..00000000000 --- a/consensus/hotstuff/forks/finalizer.go +++ /dev/null @@ -1,17 +0,0 @@ -package forks - -import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// Finalizer is responsible for block finalization. -type Finalizer interface { - VerifyBlock(*model.Block) error - IsSafeBlock(*model.Block) bool - AddBlock(*model.Block) error - GetBlock(blockID flow.Identifier) (*model.Block, bool) - GetBlocksForView(view uint64) []*model.Block - FinalizedBlock() *model.Block - LockedBlock() *model.Block -} diff --git a/consensus/hotstuff/forks/finalizer/blockcontainer.go b/consensus/hotstuff/forks/finalizer/blockcontainer.go deleted file mode 100644 index 32b400f0cb5..00000000000 --- a/consensus/hotstuff/forks/finalizer/blockcontainer.go +++ /dev/null @@ -1,20 +0,0 @@ -package finalizer - -import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// BlockContainer wraps a block to implement forest.Vertex -// In addition, it holds some additional properties for efficient processing of blocks -// by the Finalizer -type BlockContainer struct { - Block *model.Block -} - -// functions implementing forest.vertex -func (b *BlockContainer) VertexID() flow.Identifier { return b.Block.BlockID } -func (b *BlockContainer) Level() uint64 { return b.Block.View } -func (b *BlockContainer) Parent() (flow.Identifier, uint64) { - return b.Block.QC.BlockID, b.Block.QC.View -} diff --git a/consensus/hotstuff/forks/finalizer/finalizer.go b/consensus/hotstuff/forks/finalizer/finalizer.go deleted file mode 100644 index 5246175c403..00000000000 --- a/consensus/hotstuff/forks/finalizer/finalizer.go +++ /dev/null @@ -1,391 +0,0 @@ -package finalizer - -import ( - "errors" - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/forest" -) - -// Finalizer implements HotStuff finalization logic -type Finalizer struct { - notifier hotstuff.FinalizationConsumer - forest forest.LevelledForest - - finalizationCallback module.Finalizer - lastLocked *forks.BlockQC // lastLockedBlockQC is the QC that POINTS TO the the most recently locked block - lastFinalized *forks.BlockQC // lastFinalizedBlockQC is the QC that POINTS TO the most recently finalized locked block -} - -type ancestryChain struct { - block *BlockContainer - oneChain *forks.BlockQC - twoChain *forks.BlockQC - threeChain *forks.BlockQC -} - -// ErrPrunedAncestry is a sentinel error: cannot resolve ancestry of block due to pruning -var ErrPrunedAncestry = errors.New("cannot resolve pruned ancestor") - -func New(trustedRoot *forks.BlockQC, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Finalizer, error) { - if (trustedRoot.Block.BlockID != trustedRoot.QC.BlockID) || (trustedRoot.Block.View != trustedRoot.QC.View) { - return nil, model.NewConfigurationErrorf("invalid root: root qc is not pointing to root block") - } - - fnlzr := Finalizer{ - notifier: notifier, - finalizationCallback: finalizationCallback, - forest: *forest.NewLevelledForest(trustedRoot.Block.View), - lastLocked: trustedRoot, - lastFinalized: trustedRoot, - } - // verify and add root block to levelled forest - err := fnlzr.VerifyBlock(trustedRoot.Block) - if err != nil { - return nil, fmt.Errorf("invalid root block: %w", err) - } - fnlzr.forest.AddVertex(&BlockContainer{Block: trustedRoot.Block}) - fnlzr.notifier.OnBlockIncorporated(trustedRoot.Block) - return &fnlzr, nil -} - -func (r *Finalizer) LockedBlock() *model.Block { return r.lastLocked.Block } -func (r *Finalizer) LockedBlockQC() *flow.QuorumCertificate { return r.lastLocked.QC } -func (r *Finalizer) FinalizedBlock() *model.Block { return r.lastFinalized.Block } -func (r *Finalizer) FinalizedView() uint64 { return r.lastFinalized.Block.View } -func (r *Finalizer) FinalizedBlockQC() *flow.QuorumCertificate { return r.lastFinalized.QC } - -// GetBlock returns block for given ID -func (r *Finalizer) GetBlock(blockID flow.Identifier) (*model.Block, bool) { - blockContainer, hasBlock := r.forest.GetVertex(blockID) - if !hasBlock { - return nil, false - } - return blockContainer.(*BlockContainer).Block, true -} - -// GetBlock returns all known blocks for the given -func (r *Finalizer) GetBlocksForView(view uint64) []*model.Block { - vertexIterator := r.forest.GetVerticesAtLevel(view) - l := make([]*model.Block, 0, 1) // in the vast majority of cases, there will only be one proposal for a particular view - for vertexIterator.HasNext() { - v := vertexIterator.NextVertex().(*BlockContainer) - l = append(l, v.Block) - } - return l -} - -// IsKnownBlock checks whether block is known. -// UNVALIDATED: expects block to pass Finalizer.VerifyBlock(block) -func (r *Finalizer) IsKnownBlock(block *model.Block) bool { - _, hasBlock := r.forest.GetVertex(block.BlockID) - return hasBlock -} - -// IsProcessingNeeded performs basic checks whether or not block needs processing -// only considering the block's height and hash -// Returns false if any of the following conditions applies -// - block view is _below_ the most recently finalized block -// - known block -// -// UNVALIDATED: expects block to pass Finalizer.VerifyBlock(block) -func (r *Finalizer) IsProcessingNeeded(block *model.Block) bool { - if block.View < r.lastFinalized.Block.View || r.IsKnownBlock(block) { - return false - } - return true -} - -// IsSafeBlock returns true if block is safe to vote for -// (according to the definition in https://arxiv.org/abs/1803.05069v6). -// NO MODIFICATION of consensus state (read only) -// UNVALIDATED: expects block to pass Finalizer.VerifyBlock(block) -func (r *Finalizer) IsSafeBlock(block *model.Block) bool { - // According to the paper, a block is considered a safe block if - // * it extends from locked block (safety rule), - // * or the view of the parent block is higher than the view number of locked block (liveness rule). - // The two rules can be boiled down to the following: - // 1. If block.QC.View is higher than locked view, it definitely is a safe block. - // 2. If block.QC.View is lower than locked view, it definitely is not a safe block. - // 3. If block.QC.View equals to locked view: parent must be the locked block. - qc := block.QC - if qc.View > r.lastLocked.Block.View { - return true - } - if (qc.View == r.lastLocked.Block.View) && (qc.BlockID == r.lastLocked.Block.BlockID) { - return true - } - return false -} - -// ProcessBlock adds `block` to the consensus state. -// Calling this method with previously-processed blocks leaves the consensus state invariant -// (though, it will potentially cause some duplicate processing). -// UNVALIDATED: expects block to pass Finalizer.VerifyBlock(block) -func (r *Finalizer) AddBlock(block *model.Block) error { - if !r.IsProcessingNeeded(block) { - return nil - } - blockContainer := &BlockContainer{Block: block} - if err := r.checkForConflictingQCs(blockContainer.Block.QC); err != nil { - return err - } - r.checkForDoubleProposal(blockContainer) - r.forest.AddVertex(blockContainer) - err := r.updateConsensusState(blockContainer) - if err != nil { - return fmt.Errorf("updating consensus state failed: %w", err) - } - err = r.finalizationCallback.MakeValid(blockContainer.Block.BlockID) - if err != nil { - return fmt.Errorf("MakeValid fails in other component: %w", err) - } - r.notifier.OnBlockIncorporated(blockContainer.Block) - return nil -} - -// checkForConflictingQCs checks if qc conflicts with a stored Quorum Certificate. -// In case a conflicting QC is found, an ByzantineThresholdExceededError is returned. -// -// Two Quorum Certificates q1 and q2 are defined as conflicting iff: -// - q1.View == q2.View -// - q1.BlockID != q2.BlockID -// -// This means there are two Quorums for conflicting blocks at the same view. -// Per Lemma 1 from the HotStuff paper https://arxiv.org/abs/1803.05069v6, two -// conflicting QCs can exists if and onluy of the Byzantine threshold is exceeded. -func (r *Finalizer) checkForConflictingQCs(qc *flow.QuorumCertificate) error { - it := r.forest.GetVerticesAtLevel(qc.View) - for it.HasNext() { - otherBlock := it.NextVertex() // by construction, must have same view as qc.View - if qc.BlockID != otherBlock.VertexID() { - // * we have just found another block at the same view number as qc.View but with different hash - // * if this block has a child c, this child will have - // c.qc.view = parentView - // c.qc.ID != parentBlockID - // => conflicting qc - otherChildren := r.forest.GetChildren(otherBlock.VertexID()) - if otherChildren.HasNext() { - otherChild := otherChildren.NextVertex() - conflictingQC := otherChild.(*BlockContainer).Block.QC - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "conflicting QCs at view %d: %v and %v", - qc.View, qc.BlockID, conflictingQC.BlockID, - )} - } - } - } - return nil -} - -// checkForDoubleProposal checks if Block is a double proposal. In case it is, -// notifier.OnDoubleProposeDetected is triggered -func (r *Finalizer) checkForDoubleProposal(container *BlockContainer) { - it := r.forest.GetVerticesAtLevel(container.Block.View) - for it.HasNext() { - otherVertex := it.NextVertex() // by construction, must have same view as parentView - if container.VertexID() != otherVertex.VertexID() { - r.notifier.OnDoubleProposeDetected(container.Block, otherVertex.(*BlockContainer).Block) - } - } -} - -// updateConsensusState updates consensus state. -// Calling this method with previously-processed blocks leaves the consensus state invariant. -// UNVALIDATED: assumes that relevant block properties are consistent with previous blocks -func (r *Finalizer) updateConsensusState(blockContainer *BlockContainer) error { - ancestryChain, err := r.getThreeChain(blockContainer) - // We expect that getThreeChain might error with a ErrorPrunedAncestry. This error indicates that the - // 3-chain of this block reaches _beyond_ the last finalized block. It is straight forward to show: - // Lemma: Let B be a block whose 3-chain reaches beyond the last finalized block - // => B will not update the locked or finalized block - if errors.Is(err, ErrPrunedAncestry) { // blockContainer's 3-chain reaches beyond the last finalized block - // based on Lemma from above, we can skip attempting to update locked or finalized block - return nil - } - if err != nil { // otherwise, there is an unknown error that we need to escalate to the higher-level application logic - return fmt.Errorf("retrieving 3-chain ancestry failed: %w", err) - } - - r.updateLockedQc(ancestryChain) - err = r.updateFinalizedBlockQc(ancestryChain) - if err != nil { - return fmt.Errorf("updating finalized block failed: %w", err) - } - return nil -} - -// getThreeChain returns the three chain or a ErrorPrunedAncestry sentinel error -// to indicate that the 3-chain from blockContainer is (partially) pruned -func (r *Finalizer) getThreeChain(blockContainer *BlockContainer) (*ancestryChain, error) { - ancestryChain := ancestryChain{block: blockContainer} - - var err error - ancestryChain.oneChain, err = r.getNextAncestryLevel(blockContainer.Block) - if err != nil { - return nil, err - } - ancestryChain.twoChain, err = r.getNextAncestryLevel(ancestryChain.oneChain.Block) - if err != nil { - return nil, err - } - ancestryChain.threeChain, err = r.getNextAncestryLevel(ancestryChain.twoChain.Block) - if err != nil { - return nil, err - } - return &ancestryChain, nil -} - -// getNextAncestryLevel parent from forest. Returns QCBlock for the parent, -// i.e. the parent block itself and the qc pointing to the parent, i.e. block.QC(). -// If the block's parent is below the pruned view, it will error with an ErrorPrunedAncestry. -// UNVALIDATED: expects block to pass Finalizer.VerifyBlock(block) -func (r *Finalizer) getNextAncestryLevel(block *model.Block) (*forks.BlockQC, error) { - // The finalizer prunes all blocks in forest which are below the most recently finalized block. - // Hence, we have a pruned ancestry if and only if either of the following conditions applies: - // (a) if a block's parent view (i.e. block.QC.View) is below the most recently finalized block. - // (b) if a block's view is equal to the most recently finalized block. - // Caution: - // * Under normal operation, case (b) is covered by the logic for case (a) - // * However, the existence of a genesis block requires handling case (b) explicitly: - // The root block is specified and trusted by the node operator. If the root block is the - // genesis block, it might not contain a qc pointing to a parent (as there is no parent). - // In this case, condition (a) cannot be evaluated. - if (block.View <= r.lastFinalized.Block.View) || (block.QC.View < r.lastFinalized.Block.View) { - return nil, ErrPrunedAncestry - } - - parentVertex, parentBlockKnown := r.forest.GetVertex(block.QC.BlockID) - if !parentBlockKnown { - return nil, model.MissingBlockError{View: block.QC.View, BlockID: block.QC.BlockID} - } - newBlock := parentVertex.(*BlockContainer).Block - if newBlock.BlockID != block.QC.BlockID || newBlock.View != block.QC.View { - return nil, fmt.Errorf("mismatch between finalized block and QC") - } - - blockQC := forks.BlockQC{Block: newBlock, QC: block.QC} - - return &blockQC, nil -} - -// updateLockedBlock updates `lastLockedBlockQC` -// We use the locking rule from 'Event-driven HotStuff Protocol' where the condition is: -// - Consider the set S of all blocks that have a INDIRECT 2-chain on top of it -// - The 'Locked Block' is the block in S with the _highest view number_ (newest); -// -// Calling this method with previously-processed blocks leaves consensus state invariant. -func (r *Finalizer) updateLockedQc(ancestryChain *ancestryChain) { - if ancestryChain.twoChain.Block.View <= r.lastLocked.Block.View { - return - } - // update qc to newer block with any 2-chain on top of it: - r.lastLocked = ancestryChain.twoChain -} - -// updateFinalizedBlockQc updates `lastFinalizedBlockQC` -// We use the finalization rule from 'Event-driven HotStuff Protocol' where the condition is: -// - Consider the set S of all blocks that have a DIRECT 2-chain on top of it PLUS any 1-chain -// - The 'Last finalized Block' is the block in S with the _highest view number_ (newest); -// -// Calling this method with previously-processed blocks leaves consensus state invariant. -func (r *Finalizer) updateFinalizedBlockQc(ancestryChain *ancestryChain) error { - // Note: we assume that all stored blocks pass Finalizer.VerifyBlock(block); - // specifically, that Block's ViewNumber is strictly monotonously - // increasing which is enforced by LevelledForest.VerifyVertex(...) - // We denote: - // * a DIRECT 1-chain as '<-' - // * a general 1-chain as '<~' (direct or indirect) - // The rule from 'Event-driven HotStuff' for finalizing block b is - // b <- b' <- b'' <~ b* (aka a DIRECT 2-chain PLUS any 1-chain) - // where b* is the input block to this method. - // Hence, we can finalize b, if and only the viewNumber of b'' is exactly 2 higher than the view of b - b := ancestryChain.threeChain // note that b is actually not the block itself here but rather the QC pointing to it - if ancestryChain.oneChain.Block.View != b.Block.View+2 { - return nil - } - return r.finalizeUpToBlock(b.QC) -} - -// finalizeUpToBlock finalizes all blocks up to (and including) the block pointed to by `blockQC`. -// Finalization starts with the child of `lastFinalizedBlockQC` (explicitly checked); -// and calls OnFinalizedBlock on the newly finalized blocks in the respective order -func (r *Finalizer) finalizeUpToBlock(qc *flow.QuorumCertificate) error { - if qc.View < r.lastFinalized.Block.View { - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing blocks with view %d which is lower than previously finalized block at view %d", - qc.View, r.lastFinalized.Block.View, - )} - } - if qc.View == r.lastFinalized.Block.View { - // Sanity check: the previously last Finalized Block must be an ancestor of `block` - if r.lastFinalized.Block.BlockID != qc.BlockID { - return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( - "finalizing blocks at conflicting forks: %v and %v", - qc.BlockID, r.lastFinalized.Block.BlockID, - )} - } - return nil - } - // Have: qc.View > r.lastFinalizedBlockQC.View => finalizing new block - - // get Block and finalize everything up to the block's parent - blockVertex, _ := r.forest.GetVertex(qc.BlockID) // require block to resolve parent - blockContainer := blockVertex.(*BlockContainer) - err := r.finalizeUpToBlock(blockContainer.Block.QC) // finalize Parent, i.e. the block pointed to by the block's QC - if err != nil { - return err - } - - block := blockContainer.Block - if block.BlockID != qc.BlockID || block.View != qc.View { - return fmt.Errorf("mismatch between finalized block and QC") - } - - // finalize block itself: - r.lastFinalized = &forks.BlockQC{Block: block, QC: qc} - err = r.forest.PruneUpToLevel(blockContainer.Block.View) - if err != nil { - return fmt.Errorf("pruning levelled forest failed: %w", err) - } - - // notify other critical components about finalized block - err = r.finalizationCallback.MakeFinal(blockContainer.VertexID()) - if err != nil { - return fmt.Errorf("finalization error in other component: %w", err) - } - - // notify less important components about finalized block - r.notifier.OnFinalizedBlock(blockContainer.Block) - return nil -} - -// VerifyBlock checks block for validity -func (r *Finalizer) VerifyBlock(block *model.Block) error { - if block.View < r.forest.LowestLevel { - return nil - } - blockContainer := &BlockContainer{Block: block} - err := r.forest.VerifyVertex(blockContainer) - if err != nil { - return fmt.Errorf("invalid block: %w", err) - } - - // omit checking existence of parent if block at lowest non-pruned view number - if (block.View == r.forest.LowestLevel) || (block.QC.View < r.forest.LowestLevel) { - return nil - } - // for block whose parents are _not_ below the pruning height, we expect the parent to be known. - if _, isParentKnown := r.forest.GetVertex(block.QC.BlockID); !isParentKnown { // we are missing the parent - return model.MissingBlockError{ - View: block.QC.View, - BlockID: block.QC.BlockID, - } - } - return nil -} diff --git a/consensus/hotstuff/forks/forkchoice.go b/consensus/hotstuff/forks/forkchoice.go deleted file mode 100644 index eee2080d590..00000000000 --- a/consensus/hotstuff/forks/forkchoice.go +++ /dev/null @@ -1,25 +0,0 @@ -package forks - -import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// ForkChoice determines the fork-choice. -// ForkChoice directly interfaces with the Finalizer to query required information -// (such as finalized View, stored block, etc). -type ForkChoice interface { - - // AddQC adds a Quorum Certificate to Forks; - // Errors in case the block referenced by the qc is unknown. - AddQC(qc *flow.QuorumCertificate) error - - // MakeForkChoice prompts the ForkChoice to generate a fork choice for the - // current view `curView`. The fork choice is a qc that should be used for - // building the primaries block. - // - // Error return indicates incorrect usage. Processing a QC with view v - // should result in the PaceMaker being in view v+1 or larger. Hence, given - // that the current View is curView, all QCs should have view < curView - MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) -} diff --git a/consensus/hotstuff/forks/forkchoice/forks_test.go b/consensus/hotstuff/forks/forkchoice/forks_test.go deleted file mode 100644 index 77e46f3e844..00000000000 --- a/consensus/hotstuff/forks/forkchoice/forks_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package forkchoice - -import ( - "encoding/binary" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/crypto/hash" - "github.com/onflow/flow-go/model/flow" - mockfinalizer "github.com/onflow/flow-go/module/mock" -) - -// TestForks_Initialization tests that Forks correctly reports trusted Root -func TestForks_Initialization(t *testing.T) { - forks, _, _, root := initForks(t, 1) - - assert.Equal(t, forks.FinalizedView(), uint64(1)) - assert.Equal(t, forks.FinalizedBlock(), root.Block) - - assert.Equal(t, forks.GetBlocksForView(0), []*model.Block{}) - assert.Equal(t, forks.GetBlocksForView(1), []*model.Block{root.Block}) - assert.Equal(t, forks.GetBlocksForView(2), []*model.Block{}) - - b, found := forks.GetBlock(root.Block.BlockID) - assert.True(t, found, "Missing trusted Root ") - assert.Equal(t, root.Block, b) -} - -// TestForks_AddBlock verifies that Block can be added -func TestForks_AddBlock(t *testing.T) { - forks, _, notifier, root := initForks(t, 1) - - block02 := makeBlock(2, root.QC, flow.ZeroID) - notifier.On("OnBlockIncorporated", block02).Return().Once() - err := forks.AddBlock(block02) - if err != nil { - assert.Fail(t, err.Error()) - } - notifier.AssertExpectations(t) - - assert.Equal(t, forks.GetBlocksForView(2), []*model.Block{block02}) - b, found := forks.GetBlock(block02.BlockID) - assert.True(t, found) - assert.Equal(t, block02, b) -} - -// TestForks_3ChainFinalization tests happy-path direct 3-chain finalization -func TestForks_3ChainFinalization(t *testing.T) { - forks, finCallback, notifier, root := initForks(t, 1) // includes genesis block (v1) - - block2 := makeBlock(2, root.QC, flow.ZeroID) - notifier.On("OnBlockIncorporated", block2).Return().Once() - addBlock2Forks(t, block2, forks) - notifier.AssertExpectations(t) - - block3 := makeBlock(3, qc(block2.View, block2.BlockID), flow.ZeroID) - notifier.On("OnBlockIncorporated", block3).Return().Once() - notifier.On("OnQcIncorporated", block3.QC).Return().Once() - addBlock2Forks(t, block3, forks) - notifier.AssertExpectations(t) - - // creates direct 3-chain on genesis block (1), which is already finalized - block4 := makeBlock(4, qc(block3.View, block3.BlockID), flow.ZeroID) - notifier.On("OnBlockIncorporated", block4).Return().Once() - notifier.On("OnQcIncorporated", block4.QC).Return().Once() - addBlock2Forks(t, block4, forks) - notifier.AssertExpectations(t) - - // creates direct 3-chain on block (2) => finalize (2) - block5 := makeBlock(5, qc(block4.View, block4.BlockID), flow.ZeroID) - notifier.On("OnBlockIncorporated", block5).Return().Once() - notifier.On("OnQcIncorporated", block5.QC).Return().Once() - notifier.On("OnFinalizedBlock", block2).Return().Once() - finCallback.On("MakeFinal", block2.BlockID).Return(nil).Once() - finCallback.On("MakeValid", mock.Anything).Return(nil) - addBlock2Forks(t, block5, forks) - notifier.AssertExpectations(t) - finCallback.AssertExpectations(t) - - // creates direct 3-chain on block (3) => finalize (3) - block6 := makeBlock(6, qc(block5.View, block5.BlockID), flow.ZeroID) - notifier.On("OnBlockIncorporated", block6).Return().Once() - notifier.On("OnQcIncorporated", block6.QC).Return().Once() - notifier.On("OnFinalizedBlock", block3).Return().Once() - finCallback.On("MakeFinal", block3.BlockID).Return(nil).Once() - finCallback.On("MakeValid", mock.Anything).Return(nil) - addBlock2Forks(t, block6, forks) - notifier.AssertExpectations(t) - finCallback.AssertExpectations(t) -} - -func addBlock2Forks(t *testing.T, block *model.Block, forks hotstuff.Forks) { - err := forks.AddBlock(block) - if err != nil { - assert.Fail(t, err.Error()) - } - verifyStored(t, block, forks) -} - -// verifyStored verifies that block is stored in forks -func verifyStored(t *testing.T, block *model.Block, forks hotstuff.Forks) { - b, found := forks.GetBlock(block.BlockID) - assert.True(t, found) - assert.Equal(t, block, b) - - found = false - siblings := forks.GetBlocksForView(block.View) - assert.True(t, len(siblings) > 0) - for _, b := range forks.GetBlocksForView(block.View) { - if b != block { - continue - } - if found { // we already found block in slice, i.e. this is a duplicate - assert.Fail(t, fmt.Sprintf("Duplicate block: %v", block.BlockID)) - } - found = true - } - assert.True(t, found, fmt.Sprintf("Did not find block: %v", block.BlockID)) -} - -func initForks(t *testing.T, view uint64) (*forks.Forks, *mockfinalizer.Finalizer, *mocks.Consumer, *forks.BlockQC) { - notifier := &mocks.Consumer{} - finalizationCallback := &mockfinalizer.Finalizer{} - finalizationCallback.On("MakeValid", mock.Anything).Return(nil) - - // construct Finalizer - root := makeRootBlock(t, view) - notifier.On("OnBlockIncorporated", root.Block).Return().Once() - fnlzr, err := finalizer.New(root, finalizationCallback, notifier) - require.NoError(t, err) - - // construct ForkChoice - notifier.On("OnQcIncorporated", root.QC).Return().Once() - fc, err := NewNewestForkChoice(fnlzr, notifier) - require.NoError(t, err) - - notifier.AssertExpectations(t) - return forks.New(fnlzr, fc), finalizationCallback, notifier, root -} - -func makeRootBlock(t *testing.T, view uint64) *forks.BlockQC { - // construct Finalizer with Genesis Block - genesisBlock := makeBlock(view, nil, flow.ZeroID) - genesisQC := qc(view, genesisBlock.BlockID) - root := forks.BlockQC{Block: genesisBlock, QC: genesisQC} - return &root -} - -func qc(view uint64, id flow.Identifier) *flow.QuorumCertificate { - return &flow.QuorumCertificate{View: view, BlockID: id} -} - -func makeBlock(blockView uint64, blockQC *flow.QuorumCertificate, payloadHash flow.Identifier) *model.Block { - if blockQC == nil { - blockQC = qc(0, flow.Identifier{}) - } - id := computeID(blockView, blockQC, payloadHash) - return &model.Block{ - BlockID: id, - View: blockView, - QC: blockQC, - PayloadHash: payloadHash, - } -} - -// computeID is an INCOMPLETE STUB needed so we can test Forks. -// When computing the Block's ID, this implementation only considers -// the fields used by Forks. -// TODO need full implementation -func computeID(view uint64, qc *flow.QuorumCertificate, payloadHash flow.Identifier) flow.Identifier { - id := make([]byte, 0) - - viewBytes := make([]byte, 8) - binary.BigEndian.PutUint64(viewBytes, view) - id = append(id, viewBytes...) - - qcView := make([]byte, 8) - binary.BigEndian.PutUint64(qcView, qc.View) - id = append(id, qcView...) - id = append(id, qc.BlockID[:]...) - - id = append(id, payloadHash[:]...) - - var h [hash.HashLenSHA3_256]byte - hash.ComputeSHA3_256(&h, id) - - return flow.HashToID(h[:]) -} diff --git a/consensus/hotstuff/forks/forkchoice/newest.go b/consensus/hotstuff/forks/forkchoice/newest.go deleted file mode 100644 index 5a3162cbd38..00000000000 --- a/consensus/hotstuff/forks/forkchoice/newest.go +++ /dev/null @@ -1,126 +0,0 @@ -package forkchoice - -import ( - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" -) - -// NewestForkChoice implements HotStuff's original fork choice rule: -// always use the newest QC (i.e. the QC with highest view number) -type NewestForkChoice struct { - // preferredParent stores the preferred parent to build a block on top of. It contains the - // parent block as well as the QC POINTING to the parent, which can be used to build the block. - // preferredParent.QC called 'GenericQC' in 'Chained HotStuff Protocol' https://arxiv.org/abs/1803.05069v6 - preferredParent *forks.BlockQC - finalizer *finalizer.Finalizer - notifier hotstuff.Consumer -} - -// NewNewestForkChoice creates a new NewNewestForkChoice instance -func NewNewestForkChoice(finalizer *finalizer.Finalizer, notifier hotstuff.Consumer) (*NewestForkChoice, error) { - - // build the initial block-QC pair - // NOTE: I don't like this structure because it stores view and block ID in two separate places; this means - // we don't have a single field that is the source of truth, and opens the door for bugs that would otherwise - // be impossible - block := finalizer.FinalizedBlock() - qc := finalizer.FinalizedBlockQC() - if block.BlockID != qc.BlockID || block.View != qc.View { - return nil, fmt.Errorf("mismatch between finalized block and QC") - } - - blockQC := forks.BlockQC{Block: block, QC: qc} - - fc := &NewestForkChoice{ - preferredParent: &blockQC, - finalizer: finalizer, - notifier: notifier, - } - - notifier.OnQcIncorporated(qc) - - return fc, nil -} - -// MakeForkChoice prompts the ForkChoice to generate a fork choice for the -// current view `curView`. NewestForkChoice always returns the qc with the largest -// view number seen. -// It returns a qc and the block that the qc is pointing to. -// -// PREREQUISITE: -// ForkChoice cannot generate ForkChoices retroactively for past views. -// If used correctly, MakeForkChoice should only ever have processed QCs -// whose view is smaller than curView, for the following reason: -// Processing a QC with view v should result in the PaceMaker being in -// view v+1 or larger. Hence, given that the current View is curView, -// all QCs should have view < curView. -// To prevent accidental miss-usage, ForkChoices will error if `curView` -// is smaller than the view of any qc ForkChoice has seen. -// Note that tracking the view of the newest qc is for safety purposes -// and _independent_ of the fork-choice rule. -func (fc *NewestForkChoice) MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) { - choice := fc.preferredParent - if choice.Block.View >= curView { - // sanity check; - // processing a QC with view v should result in the PaceMaker being in view v+1 or larger - // Hence, given that the current View is curView, all QCs should have view < curView - return nil, nil, fmt.Errorf( - "ForkChoice selected qc with view %d which is larger than requested view %d", - choice.Block.View, curView, - ) - } - fc.notifier.OnForkChoiceGenerated(curView, choice.QC) - return choice.QC, choice.Block, nil -} - -// AddQC updates `preferredParent` according to the fork-choice rule. -// Currently, we implement 'Chained HotStuff Protocol' where the fork-choice -// rule is: "build on newest QC" -// It assumes the QC has been validated -func (fc *NewestForkChoice) AddQC(qc *flow.QuorumCertificate) error { - if qc.View <= fc.preferredParent.Block.View { - // Per construction, preferredParent.View() is always larger than the last finalized block's view. - // Hence, this check suffices to drop all QCs with qc.View <= last_finalized_block.View(). - return nil - } - - // Have qc.View > last_finalized_block.View(). Hence, block referenced by qc should be stored: - block, err := fc.ensureBlockStored(qc) - if err != nil { - return fmt.Errorf("cannot add QC: %w", err) - } - - if block.BlockID != qc.BlockID || block.View != qc.View { - return fmt.Errorf("mismatch between finalized block and QC") - } - - blockQC := forks.BlockQC{Block: block, QC: qc} - fc.preferredParent = &blockQC - fc.notifier.OnQcIncorporated(qc) - - return nil -} - -func (fc *NewestForkChoice) ensureBlockStored(qc *flow.QuorumCertificate) (*model.Block, error) { - block, haveBlock := fc.finalizer.GetBlock(qc.BlockID) - if !haveBlock { - // This should never happen and indicates an implementation bug. - // Finding the block to which the qc points to should always be possible for the folling reason: - // * Check in function AddQC guarantees: qc.View > fc.preferredParent.Block.View - // * Forks guarantees that every block's qc is also processed by ForkChoice - // => fc.preferredParent.Block.View > fc.finalizer.FinalizedBlock().View() - // (as NewestForkChoice tracks the qc with the largest view) - // => qc.View > fc.finalizer.FinalizedBlock().View() - // any block whose view is larger than the latest finalized block should be stored in finalizer - return nil, model.MissingBlockError{View: qc.View, BlockID: qc.BlockID} - } - if block.View != qc.View { - return nil, fmt.Errorf("invalid qc with mismatching view") - } - return block, nil -} diff --git a/consensus/hotstuff/forks/forkchoice/newest_test.go b/consensus/hotstuff/forks/forkchoice/newest_test.go deleted file mode 100644 index b6990bfdf68..00000000000 --- a/consensus/hotstuff/forks/forkchoice/newest_test.go +++ /dev/null @@ -1,329 +0,0 @@ -package forkchoice - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - hs "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" - mockfinalizer "github.com/onflow/flow-go/module/mock" -) - -type ViewPair struct { - qcView uint64 - blockView uint64 -} - -type Blocks struct { - blockMap map[uint64]*model.Block - blockList []*model.Block -} - -type QCs struct { - qcMap map[uint64]*flow.QuorumCertificate - qcList []*flow.QuorumCertificate -} - -// NOTATION: -// [a, b] is a block at view "b" with a QC with view "a", -// e.g., [1, 2] means a block at view "2" with an included QC for view "1" - -// HAPPY PATH -// As the leader of 6: we receive [1, 2], [2, 3], [3, 4], [4, 5] and receive enough votes to build QC for block 5. -// the fork choice should return block and qc for 5 -func TestHappyPath(t *testing.T) { - curView := uint64(6) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - preferedQC := makeQC(5, blocks.blockMap[5].BlockID) - err := f.AddQC(preferedQC) - require.NoError(t, err) - - // the fork choice should return block and qc for view 5 - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[choiceQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// NOT ENOUGH VOTES TO BUILD QC FOR LATEST FORK: fork with newest QC -// As the leader of 6: we receive [1,2], [2,3], [3,4], [4,5] but we don't receive enough votes to build a qc for block 5. -// the fork choice should return block and qc for 4 -func TestNoQcForLatestMainFork(t *testing.T) { - curView := uint64(6) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - // the fork choice should return block and qc for view 5 - preferedQC := makeQC(4, blocks.blockMap[4].BlockID) - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[preferedQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// NEWEST OVER LONGEST: HAPPY PATH: extend newest -// As the leader of 7: we receive [1,2], [2,3], [3,4], [4,5], [2,6], and receive enough votes to build QC for block 6. -// the fork choice should return block and qc for 6 -func TestNewestOverLongestHappyPath(t *testing.T) { - curView := uint64(7) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - p5 := ViewPair{2, 6} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4, p5) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - preferedQC := makeQC(6, blocks.blockMap[6].BlockID) - err := f.AddQC(preferedQC) - require.NoError(t, err) - - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[preferedQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// NEWEST OVER LONGEST: NOT ENOUGH VOTES TO BUILD ON TOP OF NEWEST FORK: fork from newest -// As the leader of 7: we receive [1,2], [2,3], [3,4], [4,5], [2,6], but we don't receive enough votes to build a qc for block 6. -// the fork choice should return block and qc for 4 -func TestNewestOverLongestForkFromNewest(t *testing.T) { - curView := uint64(7) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - p5 := ViewPair{2, 6} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4, p5) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - // the fork choice should return block and qc for view 4 - preferedQC := makeQC(4, blocks.blockMap[4].BlockID) - notifier.On("OnForkChoiceGenerated", curView, preferedQC).Return().Once() - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[preferedQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// FORK BELOW LOCKED: NOT ENOUGH VOTES TO BUILD ON TOP OF NEWEST FORK: fork from newest -// As the leader of 8: we receive [1,2], [2,3], [3,4], [4,5], [2,6], [6,7], but we don't receive enough votes to build a qc for block 7. -// the fork choice should return block and qc for 6 -func TestForkBelowLockedForkFromNewest(t *testing.T) { - curView := uint64(8) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - p5 := ViewPair{2, 6} - p6 := ViewPair{6, 7} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4, p5, p6) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - // the fork choice should return block and qc for view 6 - preferedQC := makeQC(6, blocks.blockMap[6].BlockID) - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[choiceQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// FORK BELOW LOCKED: HAPPY PATH: extend newest -// As the leader of 8: we receive [1,2], [2,3], [3,4], [4,5], [2,6], [6,7], and receive enough votes to build QC for block 7. -// the fork choice should return block and qc for 7 -func TestForkBelowLockedHappyPath(t *testing.T) { - curView := uint64(8) - - f, notifier, root := initNewestForkChoice(t, 1) // includes genesis block (v1) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - p3 := ViewPair{3, 4} - p4 := ViewPair{4, 5} - p5 := ViewPair{2, 6} - p6 := ViewPair{6, 7} - - blocks := generateBlocks(root.QC, p1, p2, p3, p4, p5, p6) - for _, block := range blocks.blockList { - err := f.AddBlock(block) - require.NoError(t, err) - } - - preferedQC := makeQC(7, blocks.blockMap[7].BlockID) - err := f.AddQC(preferedQC) - require.NoError(t, err) - - // the fork choice should return block and qc for view 7 - choiceQC, choiceBlock, err := f.MakeForkChoice(curView) - require.NoError(t, err) - require.Equal(t, preferedQC, choiceQC) - require.Equal(t, blocks.blockMap[choiceQC.View], choiceBlock) - notifier.AssertCalled(t, "OnForkChoiceGenerated", curView, preferedQC) -} - -// Verifies notification callbacks -func TestOnQcIncorporated(t *testing.T) { - notifier := &mocks.Consumer{} - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil) - - finalizationCallback := &mockfinalizer.Finalizer{} - finalizationCallback.On("MakeFinal", mock.Anything).Return(nil) - finalizationCallback.On("MakeValid", mock.Anything).Return(nil) - - // construct Finalizer - root := makeRootBlock(t, 1) - fnlzr, _ := finalizer.New(root, finalizationCallback, notifier) - - // construct ForkChoice, it will trigger OnQcIncorporated - notifier.On("OnQcIncorporated", root.QC).Return(nil) - fc, _ := NewNewestForkChoice(fnlzr, notifier) - assert.NotNil(t, fc) - - f := forks.New(fnlzr, fc) - - p1 := ViewPair{1, 2} - p2 := ViewPair{2, 3} - - blocks := generateBlocks(root.QC, p1, p2) - - for _, block := range blocks.blockList { - // each call to AddBlock will trigger 'OnQcIncorporated' - notifier.On("OnQcIncorporated", block.QC).Return(nil) - err := f.AddBlock(block) - require.NoError(t, err) - } - - preferedQC := makeQC(3, blocks.blockMap[3].BlockID) - // call to AddQC will trigger 'OnQcIncorporated' - notifier.On("OnQcIncorporated", preferedQC).Return(nil) - err := f.AddQC(preferedQC) - require.NoError(t, err) - notifier.AssertExpectations(t) -} - -func generateBlocks(rootQC *flow.QuorumCertificate, viewPairs ...ViewPair) *Blocks { - blocks := &Blocks{ - blockMap: make(map[uint64]*model.Block), - } - qcs := &QCs{ - qcMap: make(map[uint64]*flow.QuorumCertificate), - } - var lastBlockView uint64 - for _, viewPair := range viewPairs { - var qc *flow.QuorumCertificate - if viewPair.qcView == 1 { - qc = rootQC - } else { - existedQc, exists := qcs.qcMap[viewPair.qcView] - if !exists { - qc = makeQC(viewPair.qcView, blocks.blockMap[lastBlockView].BlockID) - } else { - qc = existedQc - } - } - qcs.AddQC(qc) - - block := makeBlock(viewPair.blockView, qcs.qcMap[viewPair.qcView], flow.ZeroID) - lastBlockView = block.View - blocks.AddBlock(block) - } - - return blocks -} - -func initNewestForkChoice(t *testing.T, view uint64) (hs.Forks, *mocks.Consumer, *forks.BlockQC) { - notifier := &mocks.Consumer{} - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil) - notifier.On("OnFinalizedBlock", mock.Anything).Return(nil) - notifier.On("OnForkChoiceGenerated", mock.Anything, mock.Anything).Return().Once() - - finalizationCallback := &mockfinalizer.Finalizer{} - finalizationCallback.On("MakeFinal", mock.Anything).Return(nil) - finalizationCallback.On("MakeValid", mock.Anything).Return(nil) - - // construct Finalizer - root := makeRootBlock(t, view) - fnlzr, _ := finalizer.New(root, finalizationCallback, notifier) - - // construct ForkChoice - notifier.On("OnQcIncorporated", mock.Anything).Return(nil) - fc, _ := NewNewestForkChoice(fnlzr, notifier) - - f := forks.New(fnlzr, fc) - - return f, notifier, root -} - -func makeQC(view uint64, blockID flow.Identifier) *flow.QuorumCertificate { - return &flow.QuorumCertificate{ - View: view, - BlockID: blockID, - } -} - -func (b *Blocks) AddBlock(block *model.Block) { - b.blockMap[block.View] = block - b.blockList = append(b.blockList, block) -} - -func (q *QCs) AddQC(qc *flow.QuorumCertificate) { - q.qcMap[qc.View] = qc - q.qcList = append(q.qcList, qc) -} diff --git a/consensus/hotstuff/forks/forks.go b/consensus/hotstuff/forks/forks.go index 8af62789ced..82ce3161271 100644 --- a/consensus/hotstuff/forks/forks.go +++ b/consensus/hotstuff/forks/forks.go @@ -1,85 +1,438 @@ package forks import ( + "errors" "fmt" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/forest" + "github.com/onflow/flow-go/module/mempool" ) -// Forks implements the hotstuff.Reactor API +// ErrPrunedAncestry is a sentinel error: cannot resolve ancestry of block due to pruning +var ErrPrunedAncestry = errors.New("cannot resolve pruned ancestor") + +// ancestryChain encapsulates a block, its parent (oneChain) and its grand-parent (twoChain). +// Given a chain structure like: +// +// b <~ b' <~ b* +// +// where the QC certifying b is qc_b, this data structure looks like: +// +// twoChain oneChain block +// [b<-qc_b] [b'<-qc_b'] [b*] +type ancestryChain struct { + block *BlockContainer + oneChain *BlockQC + twoChain *BlockQC +} + +// Forks enforces structural validity of the consensus state and implements +// finalization rules as defined in Jolteon consensus https://arxiv.org/abs/2106.10362 +// The same approach has later been adopted by the Diem team resulting in DiemBFT v4: +// https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf +// Forks is NOT safe for concurrent use by multiple goroutines. type Forks struct { - finalizer Finalizer - forkchoice ForkChoice + notifier hotstuff.FinalizationConsumer + forest forest.LevelledForest + + finalizationCallback module.Finalizer + newestView uint64 // newestView is the highest view of block proposal stored in Forks + lastFinalized *BlockQC // lastFinalized is the QC that POINTS TO the most recently finalized locked block } var _ hotstuff.Forks = (*Forks)(nil) -// New creates a Forks instance -func New(finalizer Finalizer, forkchoice ForkChoice) *Forks { - return &Forks{ - finalizer: finalizer, - forkchoice: forkchoice, +func New(trustedRoot *BlockQC, finalizationCallback module.Finalizer, notifier hotstuff.FinalizationConsumer) (*Forks, error) { + if (trustedRoot.Block.BlockID != trustedRoot.QC.BlockID) || (trustedRoot.Block.View != trustedRoot.QC.View) { + return nil, model.NewConfigurationErrorf("invalid root: root QC is not pointing to root block") + } + + forks := Forks{ + notifier: notifier, + finalizationCallback: finalizationCallback, + forest: *forest.NewLevelledForest(trustedRoot.Block.View), + lastFinalized: trustedRoot, + newestView: trustedRoot.Block.View, + } + + // CAUTION: instead of a proposal, we use a normal block (without `SigData` and `LastViewTC`, + // which would be possibly included in a full proposal). Per convention, we consider the + // root block as already committed and enter a higher view. + // Therefore, the root block's proposer signature and TC are irrelevant for consensus. + trustedRootProposal := &model.Proposal{ + Block: trustedRoot.Block, } + + // verify and add root block to levelled forest + err := forks.VerifyProposal(trustedRootProposal) + if err != nil { + return nil, fmt.Errorf("invalid root block: %w", err) + } + forks.forest.AddVertex(&BlockContainer{Proposal: trustedRootProposal}) + return &forks, nil } -// GetBlocksForView returns all the blocks for a certain view. -func (f *Forks) GetBlocksForView(view uint64) []*model.Block { - return f.finalizer.GetBlocksForView(view) +func (f *Forks) FinalizedBlock() *model.Block { return f.lastFinalized.Block } +func (f *Forks) FinalizedView() uint64 { return f.lastFinalized.Block.View } +func (f *Forks) NewestView() uint64 { return f.newestView } + +// GetProposal returns block for given ID +func (f *Forks) GetProposal(blockID flow.Identifier) (*model.Proposal, bool) { + blockContainer, hasBlock := f.forest.GetVertex(blockID) + if !hasBlock { + return nil, false + } + return blockContainer.(*BlockContainer).Proposal, true } -// GetBlock returns the block for the given block ID -func (f *Forks) GetBlock(id flow.Identifier) (*model.Block, bool) { - return f.finalizer.GetBlock(id) +// GetProposalsForView returns all known proposals for the given view +func (f *Forks) GetProposalsForView(view uint64) []*model.Proposal { + vertexIterator := f.forest.GetVerticesAtLevel(view) + l := make([]*model.Proposal, 0, 1) // in the vast majority of cases, there will only be one proposal for a particular view + for vertexIterator.HasNext() { + v := vertexIterator.NextVertex().(*BlockContainer) + l = append(l, v.Proposal) + } + return l } -// FinalizedBlock returns the latest finalized block -func (f *Forks) FinalizedBlock() *model.Block { - return f.finalizer.FinalizedBlock() +// AddProposal adds proposal to the consensus state. Performs verification to make sure that we don't +// add invalid proposals into consensus state. +// We assume that all blocks are fully verified. A valid block must satisfy all consistency +// requirements; otherwise we have a bug in the compliance layer. +// Expected errors during normal operations: +// - model.ByzantineThresholdExceededError - new block results in conflicting finalized blocks +func (f *Forks) AddProposal(proposal *model.Proposal) error { + err := f.VerifyProposal(proposal) + if err != nil { + if model.IsMissingBlockError(err) { + return fmt.Errorf("cannot add proposal with missing parent: %s", err.Error()) + } + // technically, this not strictly required. However, we leave this as a sanity check for now + return fmt.Errorf("cannot add invalid proposal to Forks: %w", err) + } + err = f.UnverifiedAddProposal(proposal) + if err != nil { + return fmt.Errorf("error storing proposal in Forks: %w", err) + } + + return nil } -// FinalizedView returns the view of the latest finalized block -func (f *Forks) FinalizedView() uint64 { - return f.finalizer.FinalizedBlock().View +// IsKnownBlock checks whether block is known. +// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +func (f *Forks) IsKnownBlock(block *model.Block) bool { + _, hasBlock := f.forest.GetVertex(block.BlockID) + return hasBlock } -// IsSafeBlock returns whether a block is safe to vote for. -func (f *Forks) IsSafeBlock(block *model.Block) bool { - if err := f.finalizer.VerifyBlock(block); err != nil { +// IsProcessingNeeded performs basic checks to determine whether block needs processing, +// only considering the block's height and hash. +// Returns false if any of the following conditions applies +// - block view is _below_ the most recently finalized block +// - the block already exists in the consensus state +// +// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +func (f *Forks) IsProcessingNeeded(block *model.Block) bool { + if block.View < f.lastFinalized.Block.View || f.IsKnownBlock(block) { return false } - return f.finalizer.IsSafeBlock(block) + return true } -// AddBlock passes the block to the finalizer for finalization and -// gives the QC to forkchoice for updating the preferred parent block -func (f *Forks) AddBlock(block *model.Block) error { - if err := f.finalizer.VerifyBlock(block); err != nil { - // technically, this not strictly required. However, we leave this as a sanity check for now - return fmt.Errorf("cannot add invalid block to Forks: %w", err) +// UnverifiedAddProposal adds `proposal` to the consensus state and updates the +// latest finalized block, if possible. +// Calling this method with previously-processed blocks leaves the consensus state invariant +// (though, it will potentially cause some duplicate processing). +// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +// Error returns: +// * model.ByzantineThresholdExceededError if proposal's QC conflicts with an existing QC. +// * generic error in case of unexpected bug or internal state corruption +func (f *Forks) UnverifiedAddProposal(proposal *model.Proposal) error { + if !f.IsProcessingNeeded(proposal.Block) { + return nil + } + blockContainer := &BlockContainer{Proposal: proposal} + block := blockContainer.Proposal.Block + + err := f.checkForConflictingQCs(block.QC) + if err != nil { + return err + } + f.checkForDoubleProposal(blockContainer) + f.forest.AddVertex(blockContainer) + if f.newestView < block.View { + f.newestView = block.View + } + + err = f.updateFinalizedBlockQC(blockContainer) + if err != nil { + return fmt.Errorf("updating consensus state failed: %w", err) + } + f.notifier.OnBlockIncorporated(block) + return nil +} + +// VerifyProposal checks a block for internal consistency and consistency with +// the current forest state. See forest.VerifyVertex for more detail. +// We assume that all blocks are fully verified. A valid block must satisfy all consistency +// requirements; otherwise we have a bug in the compliance layer. +// Error returns: +// - model.MissingBlockError if the parent of the input proposal does not exist in the forest +// (but is above the pruned view) +// - generic error in case of unexpected bug or internal state corruption +func (f *Forks) VerifyProposal(proposal *model.Proposal) error { + block := proposal.Block + if block.View < f.forest.LowestLevel { + return nil + } + blockContainer := &BlockContainer{Proposal: proposal} + err := f.forest.VerifyVertex(blockContainer) + if err != nil { + if forest.IsInvalidVertexError(err) { + return fmt.Errorf("cannot add proposal %x to forest: %s", block.BlockID, err.Error()) + } + return fmt.Errorf("unexpected error verifying proposal vertex: %w", err) + } + + // omit checking existence of parent if block at lowest non-pruned view number + if (block.View == f.forest.LowestLevel) || (block.QC.View < f.forest.LowestLevel) { + return nil } - err := f.finalizer.AddBlock(block) + // for block whose parents are _not_ below the pruning height, we expect the parent to be known. + if _, isParentKnown := f.forest.GetVertex(block.QC.BlockID); !isParentKnown { // we are missing the parent + return model.MissingBlockError{ + View: block.QC.View, + BlockID: block.QC.BlockID, + } + } + return nil +} + +// checkForConflictingQCs checks if QC conflicts with a stored Quorum Certificate. +// In case a conflicting QC is found, an ByzantineThresholdExceededError is returned. +// +// Two Quorum Certificates q1 and q2 are defined as conflicting iff: +// - q1.View == q2.View +// - q1.BlockID != q2.BlockID +// +// This means there are two Quorums for conflicting blocks at the same view. +// Per 'Observation 1' from the Jolteon paper https://arxiv.org/pdf/2106.10362v1.pdf, two +// conflicting QCs can exist if and only if the Byzantine threshold is exceeded. +// Error returns: +// * model.ByzantineThresholdExceededError if input QC conflicts with an existing QC. +func (f *Forks) checkForConflictingQCs(qc *flow.QuorumCertificate) error { + it := f.forest.GetVerticesAtLevel(qc.View) + for it.HasNext() { + otherBlock := it.NextVertex() // by construction, must have same view as qc.View + if qc.BlockID != otherBlock.VertexID() { + // * we have just found another block at the same view number as qc.View but with different hash + // * if this block has a child c, this child will have + // c.qc.view = parentView + // c.qc.ID != parentBlockID + // => conflicting qc + otherChildren := f.forest.GetChildren(otherBlock.VertexID()) + if otherChildren.HasNext() { + otherChild := otherChildren.NextVertex() + conflictingQC := otherChild.(*BlockContainer).Proposal.Block.QC + return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "conflicting QCs at view %d: %v and %v", + qc.View, qc.BlockID, conflictingQC.BlockID, + )} + } + } + } + return nil +} + +// checkForDoubleProposal checks if the input proposal is a double proposal. +// A double proposal occurs when two proposals with the same view exist in Forks. +// If there is a double proposal, notifier.OnDoubleProposeDetected is triggered. +func (f *Forks) checkForDoubleProposal(container *BlockContainer) { + block := container.Proposal.Block + it := f.forest.GetVerticesAtLevel(block.View) + for it.HasNext() { + otherVertex := it.NextVertex() // by construction, must have same view as parentView + if container.VertexID() != otherVertex.VertexID() { + f.notifier.OnDoubleProposeDetected(block, otherVertex.(*BlockContainer).Proposal.Block) + } + } +} + +// updateFinalizedBlockQC updates the latest finalized block, if possible. +// This function should be called every time a new block is added to Forks. +// If the new block is the head of a 2-chain satisfying the finalization rule, +// then we update Forks.lastFinalizedBlockQC to the new latest finalized block. +// Calling this method with previously-processed blocks leaves the consensus state invariant. +// UNVALIDATED: assumes that relevant block properties are consistent with previous blocks +// Error returns: +// - model.ByzantineThresholdExceededError if we are finalizing a block which is invalid to finalize. +// This either indicates a critical internal bug / data corruption, or that the network Byzantine +// threshold was exceeded, breaking the safety guarantees of HotStuff. +// - generic error in case of unexpected bug or internal state corruption +func (f *Forks) updateFinalizedBlockQC(blockContainer *BlockContainer) error { + ancestryChain, err := f.getTwoChain(blockContainer) if err != nil { - return fmt.Errorf("error storing block in Forks: %w", err) + // We expect that getTwoChain might error with a ErrPrunedAncestry. This error indicates that the + // 2-chain of this block reaches _beyond_ the last finalized block. It is straight forward to show: + // Lemma: Let B be a block whose 2-chain reaches beyond the last finalized block + // => B will not update the locked or finalized block + if errors.Is(err, ErrPrunedAncestry) { + // blockContainer's 2-chain reaches beyond the last finalized block + // based on Lemma from above, we can skip attempting to update locked or finalized block + return nil + } + if model.IsMissingBlockError(err) { + // we are missing some un-pruned ancestry of blockContainer -> indicates corrupted internal state + return fmt.Errorf("unexpected missing block while updating consensus state: %s", err.Error()) + } + return fmt.Errorf("retrieving 2-chain ancestry failed: %w", err) } - // We only process the block's QC if the block's view is larger than the last finalized block. - // By ignoring hte qc's of block's at or below the finalized view, we allow the genesis block - // to have a nil QC. - if block.View <= f.finalizer.FinalizedBlock().View { + // Note: we assume that all stored blocks pass Forks.VerifyProposal(block); + // specifically, that Proposal's ViewNumber is strictly monotonously + // increasing which is enforced by LevelledForest.VerifyVertex(...) + // We denote: + // * a DIRECT 1-chain as '<-' + // * a general 1-chain as '<~' (direct or indirect) + // Jolteon's rule for finalizing block b is + // b <- b' <~ b* (aka a DIRECT 1-chain PLUS any 1-chain) + // where b* is the head block of the ancestryChain + // Hence, we can finalize b as head of 2-chain, if and only the viewNumber of b' is exactly 1 higher than the view of b + b := ancestryChain.twoChain + if ancestryChain.oneChain.Block.View != b.Block.View+1 { return nil } - return f.AddQC(block.QC) + return f.finalizeUpToBlock(b.QC) } -// MakeForkChoice returns the block to build new block proposal from for the current view. -// the QC is the QC that points to that block. -func (f *Forks) MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) { - return f.forkchoice.MakeForkChoice(curView) +// getTwoChain returns the 2-chain for the input block container b. +// See ancestryChain for documentation on the structure of the 2-chain. +// Returns ErrPrunedAncestry if any part of the 2-chain is below the last pruned view. +// Error returns: +// - ErrPrunedAncestry if any part of the 2-chain is below the last pruned view. +// - model.MissingBlockError if any block in the 2-chain does not exist in the forest +// (but is above the pruned view) +// - generic error in case of unexpected bug or internal state corruption +func (f *Forks) getTwoChain(blockContainer *BlockContainer) (*ancestryChain, error) { + ancestryChain := ancestryChain{block: blockContainer} + + var err error + ancestryChain.oneChain, err = f.getNextAncestryLevel(blockContainer.Proposal.Block) + if err != nil { + return nil, err + } + ancestryChain.twoChain, err = f.getNextAncestryLevel(ancestryChain.oneChain.Block) + if err != nil { + return nil, err + } + return &ancestryChain, nil +} + +// getNextAncestryLevel retrieves parent from forest. Returns QCBlock for the parent, +// i.e. the parent block itself and the qc pointing to the parent, i.e. block.QC(). +// UNVALIDATED: expects block to pass Forks.VerifyProposal(block) +// Error returns: +// - ErrPrunedAncestry if the input block's parent is below the pruned view. +// - model.MissingBlockError if the parent block does not exist in the forest +// (but is above the pruned view) +// - generic error in case of unexpected bug or internal state corruption +func (f *Forks) getNextAncestryLevel(block *model.Block) (*BlockQC, error) { + // The finalizer prunes all blocks in forest which are below the most recently finalized block. + // Hence, we have a pruned ancestry if and only if either of the following conditions applies: + // (a) if a block's parent view (i.e. block.QC.View) is below the most recently finalized block. + // (b) if a block's view is equal to the most recently finalized block. + // Caution: + // * Under normal operation, case (b) is covered by the logic for case (a) + // * However, the existence of a genesis block requires handling case (b) explicitly: + // The root block is specified and trusted by the node operator. If the root block is the + // genesis block, it might not contain a qc pointing to a parent (as there is no parent). + // In this case, condition (a) cannot be evaluated. + if (block.View <= f.lastFinalized.Block.View) || (block.QC.View < f.lastFinalized.Block.View) { + return nil, ErrPrunedAncestry + } + + parentVertex, parentBlockKnown := f.forest.GetVertex(block.QC.BlockID) + if !parentBlockKnown { + return nil, model.MissingBlockError{View: block.QC.View, BlockID: block.QC.BlockID} + } + parentBlock := parentVertex.(*BlockContainer).Proposal.Block + // sanity check consistency between input block and parent + if parentBlock.BlockID != block.QC.BlockID || parentBlock.View != block.QC.View { + return nil, fmt.Errorf("parent/child mismatch while getting ancestry level: child: (id=%x, view=%d, qc.view=%d, qc.block_id=%x) parent: (id=%x, view=%d)", + block.BlockID, block.View, block.QC.View, block.QC.BlockID, parentBlock.BlockID, parentBlock.View) + } + + blockQC := BlockQC{Block: parentBlock, QC: block.QC} + + return &blockQC, nil } -// AddQC gives the QC to the forkchoice for updating the preferred parent block -func (f *Forks) AddQC(qc *flow.QuorumCertificate) error { - return f.forkchoice.AddQC(qc) // forkchoice ensures that block referenced by qc is known +// finalizeUpToBlock finalizes all blocks up to (and including) the block pointed to by `qc`. +// Finalization starts with the child of `lastFinalizedBlockQC` (explicitly checked); +// and calls OnFinalizedBlock on the newly finalized blocks in increasing height order. +// Error returns: +// - model.ByzantineThresholdExceededError if we are finalizing a block which is invalid to finalize. +// This either indicates a critical internal bug / data corruption, or that the network Byzantine +// threshold was exceeded, breaking the safety guarantees of HotStuff. +// - generic error in case of bug or internal state corruption +func (f *Forks) finalizeUpToBlock(qc *flow.QuorumCertificate) error { + if qc.View < f.lastFinalized.Block.View { + return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "finalizing blocks with view %d which is lower than previously finalized block at view %d", + qc.View, f.lastFinalized.Block.View, + )} + } + if qc.View == f.lastFinalized.Block.View { + // Sanity check: the previously last Finalized Proposal must be an ancestor of `block` + if f.lastFinalized.Block.BlockID != qc.BlockID { + return model.ByzantineThresholdExceededError{Evidence: fmt.Sprintf( + "finalizing blocks with view %d at conflicting forks: %x and %x", + qc.View, qc.BlockID, f.lastFinalized.Block.BlockID, + )} + } + return nil + } + // Have: qc.View > f.lastFinalizedBlockQC.View => finalizing new block + + // get Proposal and finalize everything up to the block's parent + blockVertex, ok := f.forest.GetVertex(qc.BlockID) // require block to resolve parent + if !ok { + return fmt.Errorf("failed to get parent while finalizing blocks (qc.view=%d, qc.block_id=%x)", qc.View, qc.BlockID) + } + blockContainer := blockVertex.(*BlockContainer) + block := blockContainer.Proposal.Block + err := f.finalizeUpToBlock(block.QC) // finalize Parent, i.e. the block pointed to by the block's QC + if err != nil { + return err + } + + if block.BlockID != qc.BlockID || block.View != qc.View { + return fmt.Errorf("mismatch between finalized block and QC") + } + + // finalize block itself: + f.lastFinalized = &BlockQC{Block: block, QC: qc} + err = f.forest.PruneUpToLevel(block.View) + if err != nil { + if mempool.IsBelowPrunedThresholdError(err) { + // we should never see this error because we finalize blocks in strictly increasing view order + return fmt.Errorf("unexpected error pruning forest, indicates corrupted state: %s", err.Error()) + } + return fmt.Errorf("unexpected error while pruning forest: %w", err) + } + + // notify other critical components about finalized block - all errors returned are considered critical + err = f.finalizationCallback.MakeFinal(blockContainer.VertexID()) + if err != nil { + return fmt.Errorf("finalization error in other component: %w", err) + } + + // notify less important components about finalized block + f.notifier.OnFinalizedBlock(block) + return nil } diff --git a/consensus/hotstuff/forks/forks_test.go b/consensus/hotstuff/forks/forks_test.go new file mode 100644 index 00000000000..0b2856ea9f3 --- /dev/null +++ b/consensus/hotstuff/forks/forks_test.go @@ -0,0 +1,499 @@ +package forks + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + mockmodule "github.com/onflow/flow-go/module/mock" +) + +// NOTATION: +// A block is denoted as [<qc_number>, <block_view_number>]. +// For example, [1,2] means: a block of view 2 has a QC for view 1. + +// TestFinalize_Direct1Chain tests adding a direct 1-chain. +// receives [1,2] [2,3] +// it should not finalize any block because there is no finalizable 2-chain. +func TestFinalize_Direct1Chain(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireNoBlocksFinalized(t, forks) +} + +// TestFinalize_Direct2Chain tests adding a direct 1-chain on a direct 1-chain (direct 2-chain). +// receives [1,2] [2,3] [3,4] +// it should finalize [1,2] +func TestFinalize_Direct2Chain(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 1, 2) +} + +// TestFinalize_DirectIndirect2Chain tests adding an indirect 1-chain on a direct 1-chain. +// receives [1,2] [2,3] [3,5] +// it should finalize [1,2] +func TestFinalize_DirectIndirect2Chain(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 5) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 1, 2) +} + +// TestFinalize_IndirectDirect2Chain tests adding a direct 1-chain on an indirect 1-chain. +// receives [1,2] [2,4] [4,5] +// it should not finalize any blocks because there is no finalizable 2-chain. +func TestFinalize_IndirectDirect2Chain(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 4) + builder.Add(4, 5) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireNoBlocksFinalized(t, forks) +} + +// TestFinalize_Direct2ChainOnIndirect tests adding a direct 2-chain on an indirect 2-chain. +// The head of highest 2-chain should be finalized. +// receives [1,3] [3,5] [5,6] [6,7] [7,8] +// it should finalize [5,6] +func TestFinalize_Direct2ChainOnIndirect(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 3) + builder.Add(3, 5) + builder.Add(5, 6) + builder.Add(6, 7) + builder.Add(7, 8) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 5, 6) +} + +// TestFinalize_Direct2ChainOnDirect tests adding a sequence of direct 2-chains. +// The head of highest 2-chain should be finalized. +// receives [1,2] [2,3] [3,4] [4,5] [5,6] +// it should finalize [3,4] +func TestFinalize_Direct2ChainOnDirect(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(4, 5) + builder.Add(5, 6) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 3, 4) +} + +// TestFinalize_Multiple2Chains tests the case where a block can be finalized +// by different 2-chains. +// receives [1,2] [2,3] [3,5] [3,6] [3,7] +// it should finalize [1,2] +func TestFinalize_Multiple2Chains(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 5) + builder.Add(3, 6) + builder.Add(3, 7) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 1, 2) +} + +// TestFinalize_OrphanedFork tests that we can finalize a block which causes +// a conflicting fork to be orphaned. +// receives [1,2] [2,3] [2,4] [4,5] [5,6] +// it should finalize [2,4] +func TestFinalize_OrphanedFork(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(2, 4) + builder.Add(4, 5) + builder.Add(5, 6) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 2, 4) +} + +// TestDuplication tests that delivering the same block/qc multiple times has +// the same end state as delivering the block/qc once. +// receives [1,2] [2,3] [2,3] [3,4] [3,4] [4,5] [4,5] +// it should finalize [2,3] +func TestDuplication(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(3, 4) + builder.Add(4, 5) + builder.Add(4, 5) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 2, 3) +} + +// TestIgnoreBlocksBelowFinalizedView tests that blocks below finalized view are ignored. +// receives [1,2] [2,3] [3,4] [1,5] +// it should finalize [1,2] +func TestIgnoreBlocksBelowFinalizedView(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(1, 5) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 1, 2) +} + +// TestDoubleProposal tests that the DoubleProposal notification is emitted when two different +// proposals for the same view are added. +// receives [1,2] [2,3] [3,4] [4,5] [3,5'] +// it should finalize block [2,3], and emits an DoubleProposal event with ([3,5'], [4,5]) +func TestDoubleProposal(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(4, 5) + builder.AddVersioned(3, 5, 0, 1) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[4].Block, blocks[3].Block).Once() + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + requireLatestFinalizedBlock(t, forks, 2, 3) +} + +// TestConflictingQCs checks that adding 2 conflicting QCs should return model.ByzantineThresholdExceededError +// receives [1,2] [2,3] [2,3'] [3,4] [3',5] +// it should return fatal error, because conflicting blocks 3 and 3' both received enough votes for QC +func TestConflictingQCs(t *testing.T) { + builder := NewBlockBuilder() + + builder.Add(1, 2) + builder.Add(2, 3) + builder.AddVersioned(2, 3, 0, 1) // make a conflicting proposal at view 3 + builder.Add(3, 4) // creates a QC for 3 + builder.AddVersioned(3, 5, 1, 0) // creates a QC for 3' + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Return(nil) + + err = addBlocksToForks(forks, blocks) + require.NotNil(t, err) + assert.True(t, model.IsByzantineThresholdExceededError(err)) +} + +// TestConflictingFinalizedForks checks that finalizing 2 conflicting forks should return model.ByzantineThresholdExceededError +// receives [1,2] [2,3] [2,6] [3,4] [4,5] [6,7] [7,8] +// It should return fatal error, because 2 conflicting forks were finalized +func TestConflictingFinalizedForks(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(4, 5) // finalizes (2,3) + builder.Add(2, 6) + builder.Add(6, 7) + builder.Add(7, 8) // finalizes (2,6) conflicts with (2,3) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + err = addBlocksToForks(forks, blocks) + require.Error(t, err) + assert.True(t, model.IsByzantineThresholdExceededError(err)) +} + +// TestAddUnconnectedProposal checks that adding a proposal which does not connect to the +// latest finalized block returns an exception. +// receives [2,3] +// should return fatal error, because the proposal is invalid for addition to Forks +func TestAddUnconnectedProposal(t *testing.T) { + unconnectedProposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(3), + ))) + + forks, _ := newForks(t) + + err := forks.AddProposal(unconnectedProposal) + require.Error(t, err) + // adding a disconnected block is an internal error, should return generic error + assert.False(t, model.IsByzantineThresholdExceededError(err)) +} + +// TestGetProposal tests that we can retrieve stored proposals. +// Attempting to retrieve nonexistent or pruned proposals should fail. +// receives [1,2] [2,3] [3,4], then [4,5] +// should finalize [1,2], then [2,3] +func TestGetProposal(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + builder.Add(4, 5) + + blocks, err := builder.Blocks() + require.Nil(t, err) + blocksAddedFirst := blocks[:3] // [1,2] [2,3] [3,4] + blocksAddedSecond := blocks[3:] // [4,5] + + forks, _ := newForks(t) + + // should be unable to retrieve a block before it is added + _, ok := forks.GetProposal(blocks[0].Block.BlockID) + assert.False(t, ok) + + // add first blocks - should finalize [1,2] + err = addBlocksToForks(forks, blocksAddedFirst) + require.Nil(t, err) + + // should be able to retrieve all stored blocks + for _, proposal := range blocksAddedFirst { + got, ok := forks.GetProposal(proposal.Block.BlockID) + assert.True(t, ok) + assert.Equal(t, proposal, got) + } + + // add second blocks - should finalize [2,3] and prune [1,2] + err = addBlocksToForks(forks, blocksAddedSecond) + require.Nil(t, err) + + // should be able to retrieve just added block + got, ok := forks.GetProposal(blocksAddedSecond[0].Block.BlockID) + assert.True(t, ok) + assert.Equal(t, blocksAddedSecond[0], got) + + // should be unable to retrieve pruned block + _, ok = forks.GetProposal(blocksAddedFirst[0].Block.BlockID) + assert.False(t, ok) +} + +// TestGetProposalsForView tests retrieving proposals for a view. +// receives [1,2] [2,4] [2,4'] +func TestGetProposalsForView(t *testing.T) { + + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 4) + builder.AddVersioned(2, 4, 0, 1) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", blocks[2].Block, blocks[1].Block).Once() + + err = addBlocksToForks(forks, blocks) + require.Nil(t, err) + + // 1 proposal at view 2 + proposals := forks.GetProposalsForView(2) + assert.Len(t, proposals, 1) + assert.Equal(t, blocks[0], proposals[0]) + + // 2 proposals at view 4 + proposals = forks.GetProposalsForView(4) + assert.Len(t, proposals, 2) + assert.ElementsMatch(t, blocks[1:], proposals) + + // 0 proposals at view 3 + proposals = forks.GetProposalsForView(3) + assert.Len(t, proposals, 0) +} + +// TestNotification tests that notifier gets correct notifications when incorporating block as well as finalization events. +// receives [1,2] [2,3] [3,4] +// should finalize [1,2] +func TestNotification(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + notifier := &mocks.Consumer{} + // 4 blocks including the genesis are incorporated + notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(4) + notifier.On("OnFinalizedBlock", blocks[0].Block).Return(nil).Once() + finalizationCallback := mockmodule.NewFinalizer(t) + finalizationCallback.On("MakeFinal", blocks[0].Block.BlockID).Return(nil).Once() + + genesisBQ := makeGenesis() + + forks, err := New(genesisBQ, finalizationCallback, notifier) + require.NoError(t, err) + + err = addBlocksToForks(forks, blocks) + require.NoError(t, err) +} + +// TestNewestView tests that Forks tracks the newest block view seen in received blocks. +// receives [1,2] [2,3] [3,4] +func TestNewestView(t *testing.T) { + builder := NewBlockBuilder() + builder.Add(1, 2) + builder.Add(2, 3) + builder.Add(3, 4) + + blocks, err := builder.Blocks() + require.Nil(t, err) + + forks, _ := newForks(t) + + genesis := makeGenesis() + + // initially newest view should be genesis block view + require.Equal(t, forks.NewestView(), genesis.Block.View) + + err = addBlocksToForks(forks, blocks) + require.NoError(t, err) + // after inserting new blocks, newest view should be greatest view of all added blocks + require.Equal(t, forks.NewestView(), uint64(4)) +} + +// ========== internal functions =============== + +func newForks(t *testing.T) (*Forks, *mocks.Consumer) { + notifier := mocks.NewConsumer(t) + notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Maybe() + notifier.On("OnFinalizedBlock", mock.Anything).Return(nil).Maybe() + finalizationCallback := mockmodule.NewFinalizer(t) + finalizationCallback.On("MakeFinal", mock.Anything).Return(nil).Maybe() + + genesisBQ := makeGenesis() + + forks, err := New(genesisBQ, finalizationCallback, notifier) + + require.Nil(t, err) + return forks, notifier +} + +// addBlocksToForks adds all the given blocks to Forks, in order. +// If any errors occur, returns the first one. +func addBlocksToForks(forks *Forks, proposals []*model.Proposal) error { + for _, proposal := range proposals { + err := forks.AddProposal(proposal) + if err != nil { + return fmt.Errorf("test case failed at adding proposal: %v: %w", proposal.Block.View, err) + } + } + + return nil +} + +// requireLatestFinalizedBlock asserts that the latest finalized block has the given view and qc view. +func requireLatestFinalizedBlock(t *testing.T, forks *Forks, qcView int, view int) { + require.Equal(t, forks.FinalizedBlock().View, uint64(view), "finalized block has wrong view") + require.Equal(t, forks.FinalizedBlock().QC.View, uint64(qcView), "finalized block has wrong qc") +} + +// requireNoBlocksFinalized asserts that no blocks have been finalized (genesis is latest finalized block). +func requireNoBlocksFinalized(t *testing.T, forks *Forks) { + genesis := makeGenesis() + require.Equal(t, forks.FinalizedBlock().View, genesis.Block.View) + require.Equal(t, forks.FinalizedBlock().View, genesis.QC.View) +} diff --git a/consensus/hotstuff/forks/test/finalizer_test.go b/consensus/hotstuff/forks/test/finalizer_test.go deleted file mode 100644 index 166a677630e..00000000000 --- a/consensus/hotstuff/forks/test/finalizer_test.go +++ /dev/null @@ -1,610 +0,0 @@ -package test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - mockm "github.com/onflow/flow-go/module/mock" -) - -// denotion: -// A block is denoted as [<qc_number>, <block_view_number>]. -// For example, [1,2] means: a block of view 2 has a QC for view 1. - -// receives [1,2], [2,3], [3,4], [4,5], -// it should finalize [1,2], it should lock [2,3]. -func TestLocked(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) // creates a block of view 2, with a QC of view 1 - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) // check if the finalized block has view 1, and its QC is 2 - assertTheLockedBlock(t, fin, 2, 3) - // ^^^ the reason it's not called "assertLockedBlock" is to match - // its length with assertFinalizedBlock in order to align their arguments -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], [6,8] -// it should finalize [1,2], it should lock [3,4]. -func TestLocked2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 6) - builder.Add(6, 8) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 3, 4) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], [6,8], [8,10] -// it should finalize [1,2], it should lock [4,6]. -func TestLocked3(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 6) - builder.Add(6, 8) - builder.Add(8, 10) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 4, 6) -} - -// receives [1,2], [2,3], [3,4], [4,5], [5,6] -// it should finalize [2,3], it should lock [3,4] -func TestFinalizedDirect3builder(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 2, 3) - assertTheLockedBlock(t, fin, 3, 4) -} - -// receives [1,2], [2,3], [3,4], [4,5], [5,6], [6,7], [7,8], [8, 9] -// it should finalize [5,6], it should lock [6,7] -func TestFinalizedDirect3builder2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 6) - builder.Add(6, 7) - builder.Add(7, 8) - builder.Add(8, 9) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 5, 6) - assertTheLockedBlock(t, fin, 6, 7) -} - -// receives [1,2], [2,3], [3,4], [4,5], [5,7], -// it should finalize [2,3], it should lock [3,4] -func TestFinalizedDirect2builderPlus1builder(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 7) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 2, 3) - assertTheLockedBlock(t, fin, 3, 4) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], -// it should finalize [1,2], it should lock [2,3] -func TestUnfinalized(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 3) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,7], -// it should finalize [1,2], it should lock [2,3] -func TestUnfinalized2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 7) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 3) -} - -// Tolerable Forks that extend from locked block (1: might change locked block, 2: not change locked block) -// receives [1,2], [2,3], [3,4], [4,5], [3,6], [6,7], [7,8] -// it should finalize [1,2], it should lock [3,6] -func TestTolerableForksExtendsFromLockedBlock(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(3, 6) - builder.Add(6, 7) - builder.Add(7, 8) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 3, 6) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], [6,7], [7,8] -// it should finalize [1,2], it should lock [4,6] -func TestTolerableForksExtendsFromLockedBlock2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 6) - builder.Add(6, 7) - builder.Add(7, 8) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 4, 6) -} - -// receives [1,2], [2,3], [3,4], [4,5], [3,6], [6,7], [7,8], [8,9] -// it should finalize [3,6], it should lock [6,7] -func TestTolerableForksExtendsFromLockedBlock3(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(3, 6) - builder.Add(6, 7) - builder.Add(7, 8) - builder.Add(8, 9) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 3, 6) - assertTheLockedBlock(t, fin, 6, 7) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], [6,7], [7,8], [8,9] -// it should finalize [4,6], it should lock [6,7] -func TestTolerableForksExtendsFromLockedBlock4(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(4, 6) - builder.Add(6, 7) - builder.Add(7, 8) - builder.Add(8, 9) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 4, 6) - assertTheLockedBlock(t, fin, 6, 7) -} - -// receives [1,2], [2,3], [3,4], [4,5], [4,6], [6,7], [7,8], [8,10] -// it should finalize [3,6], it should lock [6,7] -func TestTolerableForksExtendsFromLockedBlock5(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(3, 6) - builder.Add(6, 7) - builder.Add(7, 8) - builder.Add(8, 10) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 3, 6) - assertTheLockedBlock(t, fin, 6, 7) -} - -// receives [1,2], [2,3], [3,4], [4,5], [2,6] -// it should finalize [1,2], it should lock [2,3] -func TestTolerableForksNotExtendsFromLockedBlock(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(2, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 3) -} - -// receives [1,2], [2,3], [3,4], [4,5], [2,6], [5,6] -// it should finalize [2,3], it should lock [3,4], because [2,6] is replaced by [5,6] -func TestTolerableForksNotExtendsFromLockedBlock2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(2, 6) - builder.Add(5, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, notifier, _ := newFinalizer(t) - notifier.On("OnDoubleProposeDetected", blocks[5], blocks[4]).Return(nil) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 2, 3) - assertTheLockedBlock(t, fin, 3, 4) - notifier.AssertExpectations(t) -} - -// receives [1,2], [2,3], [3,4], [4,5], [2,6], [6,7] -// it should finalize [1,2], it should lock [2,3] -func TestTolerableForksNotExtendsFromLockedBlock3(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(2, 6) - builder.Add(6, 7) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 3) -} - -// receives [1,2], [2,3], [3,4], [4,5], [2,6], [6,7],[7,8] -// it should finalize [1,2], it should lock [2,6] -func TestTolerableForksNotExtendsFromLockedBlock4(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(2, 6) - builder.Add(6, 7) - builder.Add(7, 8) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 6) -} - -// receives [1,2], [2,3], [2,3], [3,4], [3,4], [4,5], [4,5], [5,6], [5,6] -// it should finalize [2,3], it should lock [3,4] -func TestDuplication(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 6) - builder.Add(4, 5) - builder.Add(5, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 2, 3) - assertTheLockedBlock(t, fin, 3, 4) -} - -// receives [1,2], [2,3], [3,4], [4,5], [1,6] -// it should finalize [1,2], it should lock [2,3] -func TestIgnoreBlocksBelowFinalizedView(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(1, 6) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 1, 2) - assertTheLockedBlock(t, fin, 2, 3) -} - -// receives [1,2], [2,3], [3,4], [4,5], [3,6], [5,6']. -// it should finalize block [2,3], and emits an DoubleProposal event with ([3,6], [5,6']) -func TestDoubleProposal(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(3, 6) - builder.AddVersioned(5, 6, 0, 1) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, notifier, _ := newFinalizer(t) - notifier.On("OnDoubleProposeDetected", blocks[5], blocks[4]).Return(nil) - - err = addBlocksToFinalizer(fin, blocks) - require.Nil(t, err) - - assertFinalizedBlock(t, fin, 2, 3) - notifier.AssertExpectations(t) -} - -// receives [1,2], [2,3], [3,4], [3,4'], [4,5], [4',6]. -// it should return fatal error, because conflicting blocks 4 and 4' -// both received enough votes for QC -func TestUntolerableForks(t *testing.T) { - builder := NewBlockBuilder() - - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.AddVersioned(3, 4, 0, 1) // make a special view 4 - builder.Add(4, 5) - builder.AddVersioned(4, 6, 1, 0) // make a special view 6 extends from special view 4 - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, notifier, _ := newFinalizer(t) - notifier.On("OnDoubleProposeDetected", blocks[3], blocks[2]).Return(nil) - - err = addBlocksToFinalizer(fin, blocks) - require.NotNil(t, err) - notifier.AssertExpectations(t) -} - -// receives [1,2], [2,3], [2,7], [3,4], [4,5], [5,6], [7,8], [8,9], [9,10] -// It should return fatal error, because a fork below locked block got finalized -func TestUntolerableForks2(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - builder.Add(5, 6) // this finalizes (2,3) - builder.Add(2, 7) - builder.Add(7, 8) - builder.Add(8, 9) - builder.Add(9, 10) // this finalizes (2,7), which is a conflicting fork with (2,3) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - fin, _, _ := newFinalizer(t) - - err = addBlocksToFinalizer(fin, blocks) - assert.Error(t, err) -} - -func TestNotification(t *testing.T) { - builder := NewBlockBuilder() - builder.Add(1, 2) - builder.Add(2, 3) - builder.Add(3, 4) - builder.Add(4, 5) - - blocks, err := builder.Blocks() - require.Nil(t, err) - - notifier := &mocks.Consumer{} - // 5 blocks including the genesis are incorporated - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil).Times(5) - notifier.On("OnFinalizedBlock", blocks[0]).Return(nil).Once() - finalizationCallback := &mockm.Finalizer{} - finalizationCallback.On("MakeFinal", blocks[0].BlockID).Return(nil).Once() - finalizationCallback.On("MakeValid", mock.Anything).Return(nil) - - genesisBQ := makeGenesis() - - fin, err := finalizer.New(genesisBQ, finalizationCallback, notifier) - require.NoError(t, err) - - err = addBlocksToFinalizer(fin, blocks) - require.NoError(t, err) - notifier.AssertExpectations(t) - finalizationCallback.AssertExpectations(t) -} - -// ========== internal functions =============== - -func newFinalizer(t *testing.T) (forks.Finalizer, *mocks.Consumer, *mockm.Finalizer) { - notifier := &mocks.Consumer{} - notifier.On("OnBlockIncorporated", mock.Anything).Return(nil) - notifier.On("OnFinalizedBlock", mock.Anything).Return(nil) - finalizationCallback := &mockm.Finalizer{} - finalizationCallback.On("MakeFinal", mock.Anything).Return(nil) - finalizationCallback.On("MakeValid", mock.Anything).Return(nil) - - genesisBQ := makeGenesis() - - fin, err := finalizer.New(genesisBQ, finalizationCallback, notifier) - - require.Nil(t, err) - return fin, notifier, finalizationCallback -} - -func addBlocksToFinalizer(fin forks.Finalizer, blocks []*model.Block) error { - for _, block := range blocks { - err := fin.AddBlock(block) - if err != nil { - return fmt.Errorf("test case failed at adding block: %v: %w", block.View, err) - } - } - - return nil -} - -// check the view and QC's view of the locked block for the finalizer -func assertTheLockedBlock(t *testing.T, fin forks.Finalizer, qc int, view int) { - assert.Equal(t, fin.LockedBlock().View, uint64(view), "locked block has wrong view") - assert.Equal(t, fin.LockedBlock().QC.View, uint64(qc), "locked block has wrong qc") -} - -// check the view and QC's view of the finalized block for the finalizer -func assertFinalizedBlock(t *testing.T, fin forks.Finalizer, qc int, view int) { - assert.Equal(t, fin.FinalizedBlock().View, uint64(view), "finalized block has wrong view") - assert.Equal(t, fin.FinalizedBlock().QC.View, uint64(qc), "fianlized block has wrong qc") -} diff --git a/consensus/hotstuff/helper/block.go b/consensus/hotstuff/helper/block.go index 7e029f8b58b..a3fa6f6e2e7 100644 --- a/consensus/hotstuff/helper/block.go +++ b/consensus/hotstuff/helper/block.go @@ -50,6 +50,12 @@ func WithParentSigners(signerIndices []byte) func(*model.Block) { } } +func WithBlockQC(qc *flow.QuorumCertificate) func(*model.Block) { + return func(block *model.Block) { + block.QC = qc + } +} + func MakeProposal(options ...func(*model.Proposal)) *model.Proposal { proposal := &model.Proposal{ Block: MakeBlock(), @@ -72,3 +78,9 @@ func WithSigData(sigData []byte) func(*model.Proposal) { proposal.SigData = sigData } } + +func WithLastViewTC(lastViewTC *flow.TimeoutCertificate) func(*model.Proposal) { + return func(proposal *model.Proposal) { + proposal.LastViewTC = lastViewTC + } +} diff --git a/consensus/hotstuff/helper/timeout_certificate.go b/consensus/hotstuff/helper/timeout_certificate.go new file mode 100644 index 00000000000..42ffe64d8d9 --- /dev/null +++ b/consensus/hotstuff/helper/timeout_certificate.go @@ -0,0 +1,94 @@ +package helper + +import ( + "math/rand" + + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func MakeTC(options ...func(*flow.TimeoutCertificate)) *flow.TimeoutCertificate { + qc := MakeQC() + signerIndices := unittest.SignerIndicesFixture(3) + highQCViews := make([]uint64, 3) + for i := range highQCViews { + highQCViews[i] = qc.View + } + tc := flow.TimeoutCertificate{ + View: rand.Uint64(), + NewestQC: qc, + NewestQCViews: []uint64{qc.View}, + SignerIndices: signerIndices, + SigData: unittest.SignatureFixture(), + } + for _, option := range options { + option(&tc) + } + return &tc +} + +func WithTCNewestQC(qc *flow.QuorumCertificate) func(*flow.TimeoutCertificate) { + return func(tc *flow.TimeoutCertificate) { + tc.NewestQC = qc + tc.NewestQCViews = []uint64{qc.View} + } +} + +func WithTCSigners(signerIndices []byte) func(*flow.TimeoutCertificate) { + return func(tc *flow.TimeoutCertificate) { + tc.SignerIndices = signerIndices + } +} + +func WithTCView(view uint64) func(*flow.TimeoutCertificate) { + return func(tc *flow.TimeoutCertificate) { + tc.View = view + } +} + +func WithTCHighQCViews(highQCViews []uint64) func(*flow.TimeoutCertificate) { + return func(tc *flow.TimeoutCertificate) { + tc.NewestQCViews = highQCViews + } +} + +func TimeoutObjectFixture(opts ...func(TimeoutObject *hotstuff.TimeoutObject)) *hotstuff.TimeoutObject { + timeout := &hotstuff.TimeoutObject{ + View: uint64(rand.Uint32()), + NewestQC: MakeQC(), + LastViewTC: MakeTC(), + SignerID: unittest.IdentifierFixture(), + SigData: unittest.RandomBytes(128), + } + + for _, opt := range opts { + opt(timeout) + } + + return timeout +} + +func WithTimeoutObjectSignerID(signerID flow.Identifier) func(*hotstuff.TimeoutObject) { + return func(TimeoutObject *hotstuff.TimeoutObject) { + TimeoutObject.SignerID = signerID + } +} + +func WithTimeoutNewestQC(newestQC *flow.QuorumCertificate) func(*hotstuff.TimeoutObject) { + return func(timeout *hotstuff.TimeoutObject) { + timeout.NewestQC = newestQC + } +} + +func WithTimeoutLastViewTC(lastViewTC *flow.TimeoutCertificate) func(*hotstuff.TimeoutObject) { + return func(timeout *hotstuff.TimeoutObject) { + timeout.LastViewTC = lastViewTC + } +} + +func WithTimeoutObjectView(view uint64) func(*hotstuff.TimeoutObject) { + return func(TimeoutObject *hotstuff.TimeoutObject) { + TimeoutObject.View = view + } +} diff --git a/consensus/hotstuff/integration/connect_test.go b/consensus/hotstuff/integration/connect_test.go index ef49bf04d7e..a254e0f9f3c 100644 --- a/consensus/hotstuff/integration/connect_test.go +++ b/consensus/hotstuff/integration/connect_test.go @@ -1,17 +1,16 @@ package integration import ( - "fmt" - "time" + "testing" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) -func Connect(instances []*Instance) { +func Connect(t *testing.T, instances []*Instance) { // first, create a map of all instances and a queue for each lookup := make(map[flow.Identifier]*Instance) @@ -23,29 +22,29 @@ func Connect(instances []*Instance) { for _, sender := range instances { sender := sender // avoid capturing loop variable in closure - *sender.communicator = mocks.Communicator{} - sender.communicator.On("BroadcastProposalWithDelay", mock.Anything, mock.Anything).Return( - func(header *flow.Header, delay time.Duration) error { + *sender.notifier = *NewMockedCommunicatorConsumer() + sender.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(t, ok) // sender should always have the parent - parent, exists := sender.headers[header.ParentID] + sender.updatingBlocks.RLock() + _, exists := sender.headers[header.ParentID] + sender.updatingBlocks.RUnlock() if !exists { - return fmt.Errorf("parent for proposal not found (sender: %x, parent: %x)", sender.localID, header.ParentID) + t.Fatalf("parent for proposal not found (sender: %x, parent: %x)", sender.localID, header.ParentID) } - // fill in the header chain ID and height - header.ChainID = parent.ChainID - header.Height = parent.Height + 1 - // convert into proposal immediately - proposal := model.ProposalFromFlow(header, parent.View) + proposal := model.ProposalFromFlow(header) // store locally and loop back to engine for processing sender.ProcessBlock(proposal) // check if we should block the outgoing proposal if sender.blockPropOut(proposal) { - return nil + return } // iterate through potential receivers @@ -58,47 +57,73 @@ func Connect(instances []*Instance) { // check if we should block the incoming proposal if receiver.blockPropIn(proposal) { - return nil + continue } receiver.ProcessBlock(proposal) } - - return nil }, ) - sender.communicator.On("SendVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return( - func(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) error { - + sender.notifier.On("OnOwnVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + blockID, ok := args[0].(flow.Identifier) + require.True(t, ok) + view, ok := args[1].(uint64) + require.True(t, ok) + sigData, ok := args[2].([]byte) + require.True(t, ok) + recipientID, ok := args[3].(flow.Identifier) + require.True(t, ok) // convert into vote vote := model.VoteFromFlow(sender.localID, blockID, view, sigData) - // should never send to self - if recipientID == sender.localID { - return fmt.Errorf("can't send to self (sender: %x)", sender.localID) - } - - // check if we should block the outgoing vote - if sender.blockVoteOut(vote) { - return nil - } - // get the receiver receiver, exists := lookup[recipientID] if !exists { - return fmt.Errorf("recipient doesn't exist (sender: %x, receiver: %x)", sender.localID, recipientID) + t.Fatalf("recipient doesn't exist (sender: %x, receiver: %x)", sender.localID, recipientID) } - // check if e should block the incoming vote - if receiver.blockVoteIn(vote) { - return nil + // if we are next leader we should be receiving our own vote + if recipientID != sender.localID { + // check if we should block the outgoing vote + if sender.blockVoteOut(vote) { + return + } + + // check if e should block the incoming vote + if receiver.blockVoteIn(vote) { + return + } } // submit the vote to the receiving event loop (non-blocking) receiver.queue <- vote - - return nil }, ) + sender.notifier.On("OnOwnTimeout", mock.Anything).Run( + func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(t, ok) + // iterate through potential receivers + for _, receiver := range instances { + + // we should skip ourselves always + if receiver.localID == sender.localID { + continue + } + + // check if we should block the outgoing value + if sender.blockTimeoutObjectOut(timeoutObject) { + continue + } + + // check if we should block the incoming value + if receiver.blockTimeoutObjectIn(timeoutObject) { + continue + } + + receiver.queue <- timeoutObject + } + }) } } diff --git a/consensus/hotstuff/integration/defaults_test.go b/consensus/hotstuff/integration/defaults_test.go index e383c13fc97..925fd3d2ec1 100644 --- a/consensus/hotstuff/integration/defaults_test.go +++ b/consensus/hotstuff/integration/defaults_test.go @@ -17,15 +17,3 @@ func DefaultRoot() *flow.Header { } return header } - -func DefaultStart() uint64 { - return 1 -} - -func DefaultPruned() uint64 { - return 0 -} - -func DefaultVoted() uint64 { - return 0 -} diff --git a/consensus/hotstuff/integration/filters_test.go b/consensus/hotstuff/integration/filters_test.go index 28e27eb6cf9..8d6ac067f48 100644 --- a/consensus/hotstuff/integration/filters_test.go +++ b/consensus/hotstuff/integration/filters_test.go @@ -7,6 +7,9 @@ import ( "github.com/onflow/flow-go/model/flow" ) +// VoteFilter is a filter function for dropping Votes. +// Return value `true` implies that the the given Vote should be +// dropped, while `false` indicates that the Vote should be received. type VoteFilter func(*model.Vote) bool func BlockNoVotes(*model.Vote) bool { @@ -17,9 +20,10 @@ func BlockAllVotes(*model.Vote) bool { return true } -func BlockVoteRatio(ratio float64) VoteFilter { +// BlockVoteRandomly drops votes randomly with a probability of `dropProbability` ∈ [0,1] +func BlockVoteRandomly(dropProbability float64) VoteFilter { return func(*model.Vote) bool { - return rand.Float64() <= ratio + return rand.Float64() < dropProbability } } @@ -29,6 +33,9 @@ func BlockVotesBy(voterID flow.Identifier) VoteFilter { } } +// ProposalFilter is a filter function for dropping Proposals. +// Return value `true` implies that the the given Proposal should be +// dropped, while `false` indicates that the Proposal should be received. type ProposalFilter func(*model.Proposal) bool func BlockNoProposals(*model.Proposal) bool { @@ -39,14 +46,31 @@ func BlockAllProposals(*model.Proposal) bool { return true } -func BlockProposalRatio(ratio float64) ProposalFilter { +// BlockProposalRandomly drops proposals randomly with a probability of `dropProbability` ∈ [0,1] +func BlockProposalRandomly(dropProbability float64) ProposalFilter { return func(*model.Proposal) bool { - return rand.Float64() <= ratio + return rand.Float64() < dropProbability } } +// BlockProposalsBy drops all proposals originating from the specified `proposerID` func BlockProposalsBy(proposerID flow.Identifier) ProposalFilter { return func(proposal *model.Proposal) bool { return proposal.Block.ProposerID == proposerID } } + +// TimeoutObjectFilter is a filter function for dropping TimeoutObjects. +// Return value `true` implies that the the given TimeoutObject should be +// dropped, while `false` indicates that the TimeoutObject should be received. +type TimeoutObjectFilter func(*model.TimeoutObject) bool + +// BlockAllTimeoutObjects always returns `true`, i.e. drops all TimeoutObjects +func BlockAllTimeoutObjects(*model.TimeoutObject) bool { + return true +} + +// BlockNoTimeoutObjects always returns `false`, i.e. it lets all TimeoutObjects pass. +func BlockNoTimeoutObjects(*model.TimeoutObject) bool { + return false +} diff --git a/consensus/hotstuff/integration/instance_test.go b/consensus/hotstuff/integration/instance_test.go index d3677abbfca..68aa714d1ba 100644 --- a/consensus/hotstuff/integration/instance_test.go +++ b/consensus/hotstuff/integration/instance_test.go @@ -4,46 +4,56 @@ import ( "context" "fmt" "sync" + "testing" "time" "github.com/gammazero/workerpool" "github.com/rs/zerolog" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "go.uber.org/atomic" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/blockproducer" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/eventhandler" "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/forks/forkchoice" "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications" + "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" + "github.com/onflow/flow-go/consensus/hotstuff/safetyrules" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutaggregator" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutcollector" "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/voteaggregator" "github.com/onflow/flow-go/consensus/hotstuff/votecollector" - "github.com/onflow/flow-go/consensus/hotstuff/voter" + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" msig "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/utils/unittest" ) type Instance struct { // instance parameters - participants flow.IdentityList - localID flow.Identifier - blockVoteIn VoteFilter - blockVoteOut VoteFilter - blockPropIn ProposalFilter - blockPropOut ProposalFilter - stop Condition + participants flow.IdentityList + localID flow.Identifier + blockVoteIn VoteFilter + blockVoteOut VoteFilter + blockPropIn ProposalFilter + blockPropOut ProposalFilter + blockTimeoutObjectIn TimeoutObjectFilter + blockTimeoutObjectOut TimeoutObjectFilter + stop Condition // instance data queue chan interface{} @@ -52,42 +62,60 @@ type Instance struct { pendings map[flow.Identifier]*model.Proposal // indexed by parent ID // mocked dependencies - committee *mocks.Committee - builder *module.Builder - finalizer *module.Finalizer - persist *mocks.Persister - signer *mocks.Signer - verifier *mocks.Verifier - communicator *mocks.Communicator + committee *mocks.DynamicCommittee + builder *module.Builder + finalizer *module.Finalizer + persist *mocks.Persister + signer *mocks.Signer + verifier *mocks.Verifier + notifier *MockedCommunicatorConsumer // real dependencies - pacemaker hotstuff.PaceMaker - producer *blockproducer.BlockProducer - forks *forks.Forks - aggregator *voteaggregator.VoteAggregator - voter *voter.Voter - validator *validator.Validator + pacemaker hotstuff.PaceMaker + producer *blockproducer.BlockProducer + forks *forks.Forks + voteAggregator *voteaggregator.VoteAggregator + timeoutAggregator *timeoutaggregator.TimeoutAggregator + safetyRules *safetyrules.SafetyRules + validator *validator.Validator // main logic handler *eventhandler.EventHandler } -func NewInstance(t require.TestingT, options ...Option) *Instance { +type MockedCommunicatorConsumer struct { + notifications.NoopPartialConsumer + notifications.NoopFinalizationConsumer + *mocks.CommunicatorConsumer +} + +func NewMockedCommunicatorConsumer() *MockedCommunicatorConsumer { + return &MockedCommunicatorConsumer{ + CommunicatorConsumer: &mocks.CommunicatorConsumer{}, + } +} + +var _ hotstuff.Consumer = (*MockedCommunicatorConsumer)(nil) +var _ hotstuff.TimeoutCollectorConsumer = (*Instance)(nil) + +func NewInstance(t *testing.T, options ...Option) *Instance { // generate random default identity identity := unittest.IdentityFixture() // initialize the default configuration cfg := Config{ - Root: DefaultRoot(), - Participants: flow.IdentityList{identity}, - LocalID: identity.NodeID, - Timeouts: timeout.DefaultConfig, - IncomingVotes: BlockNoVotes, - OutgoingVotes: BlockNoVotes, - IncomingProposals: BlockNoProposals, - OutgoingProposals: BlockNoProposals, - StopCondition: RightAway, + Root: DefaultRoot(), + Participants: flow.IdentityList{identity}, + LocalID: identity.NodeID, + Timeouts: timeout.DefaultConfig, + IncomingVotes: BlockNoVotes, + OutgoingVotes: BlockNoVotes, + IncomingProposals: BlockNoProposals, + OutgoingProposals: BlockNoProposals, + IncomingTimeoutObjects: BlockNoTimeoutObjects, + OutgoingTimeoutObjects: BlockNoTimeoutObjects, + StopCondition: RightAway, } // apply the custom options @@ -111,13 +139,15 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { in := Instance{ // instance parameters - participants: cfg.Participants, - localID: cfg.LocalID, - blockVoteIn: cfg.IncomingVotes, - blockVoteOut: cfg.OutgoingVotes, - blockPropIn: cfg.IncomingProposals, - blockPropOut: cfg.OutgoingProposals, - stop: cfg.StopCondition, + participants: cfg.Participants, + localID: cfg.LocalID, + blockVoteIn: cfg.IncomingVotes, + blockVoteOut: cfg.OutgoingVotes, + blockPropIn: cfg.IncomingProposals, + blockPropOut: cfg.OutgoingProposals, + blockTimeoutObjectIn: cfg.IncomingTimeoutObjects, + blockTimeoutObjectOut: cfg.OutgoingTimeoutObjects, + stop: cfg.StopCondition, // instance data pendings: make(map[flow.Identifier]*model.Proposal), @@ -125,27 +155,28 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { queue: make(chan interface{}, 1024), // instance mocks - committee: &mocks.Committee{}, - builder: &module.Builder{}, - persist: &mocks.Persister{}, - signer: &mocks.Signer{}, - verifier: &mocks.Verifier{}, - communicator: &mocks.Communicator{}, - finalizer: &module.Finalizer{}, + committee: &mocks.DynamicCommittee{}, + builder: &module.Builder{}, + persist: &mocks.Persister{}, + signer: &mocks.Signer{}, + verifier: &mocks.Verifier{}, + notifier: NewMockedCommunicatorConsumer(), + finalizer: &module.Finalizer{}, } // insert root block into headers register in.headers[cfg.Root.ID()] = cfg.Root // program the hotstuff committee state - in.committee.On("Identities", mock.Anything).Return( - func(blockID flow.Identifier) flow.IdentityList { + in.committee.On("IdentitiesByEpoch", mock.Anything).Return( + func(_ uint64) flow.IdentityList { return in.participants }, nil, ) for _, participant := range in.participants { - in.committee.On("Identity", mock.Anything, participant.NodeID).Return(participant, nil) + in.committee.On("IdentityByBlock", mock.Anything, participant.NodeID).Return(participant, nil) + in.committee.On("IdentityByEpoch", mock.Anything, participant.NodeID).Return(participant, nil) } in.committee.On("Self").Return(in.localID) in.committee.On("LeaderForView", mock.Anything).Return( @@ -153,6 +184,8 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { return in.participants[int(view)%len(in.participants)].NodeID }, nil, ) + in.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(in.participants.TotalWeight()), nil) + in.committee.On("TimeoutThresholdForView", mock.Anything).Return(committees.WeightThresholdToTimeout(in.participants.TotalWeight()), nil) // program the builder module behaviour in.builder.On("BuildOn", mock.Anything, mock.Anything).Return( @@ -167,6 +200,7 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { header := &flow.Header{ ChainID: "chain", ParentID: parentID, + ParentView: parent.View, Height: parent.Height + 1, PayloadHash: unittest.IdentifierFixture(), Timestamp: time.Now().UTC(), @@ -187,8 +221,8 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { ) // check on stop condition, stop the tests as soon as entering a certain view - in.persist.On("PutStarted", mock.Anything).Return(nil) - in.persist.On("PutVoted", mock.Anything).Return(nil) + in.persist.On("PutSafetyData", mock.Anything).Return(nil) + in.persist.On("PutLivenessData", mock.Anything).Return(nil) // program the hotstuff signer behaviour in.signer.On("CreateProposal", mock.Anything).Return( @@ -213,6 +247,19 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { }, nil, ) + in.signer.On("CreateTimeout", mock.Anything, mock.Anything, mock.Anything).Return( + func(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) *model.TimeoutObject { + timeoutObject := &model.TimeoutObject{ + View: curView, + NewestQC: newestQC, + LastViewTC: lastViewTC, + SignerID: in.localID, + SigData: unittest.RandomBytes(msig.SigLen), + } + return timeoutObject + }, + nil, + ) in.signer.On("CreateQC", mock.Anything).Return( func(votes []*model.Vote) *flow.QuorumCertificate { voterIDs := make(flow.IdentifierList, 0, len(votes)) @@ -235,36 +282,48 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { ) // program the hotstuff verifier behaviour - in.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything).Return(nil) - in.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything).Return(nil) + in.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + in.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + in.verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) // program the hotstuff communicator behaviour - in.communicator.On("BroadcastProposalWithDelay", mock.Anything, mock.Anything).Return( - func(header *flow.Header, delay time.Duration) error { + in.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run( + func(args mock.Arguments) { + header, ok := args[0].(*flow.Header) + require.True(t, ok) // sender should always have the parent in.updatingBlocks.RLock() - parent, exists := in.headers[header.ParentID] + _, exists := in.headers[header.ParentID] in.updatingBlocks.RUnlock() if !exists { - return fmt.Errorf("parent for proposal not found (sender: %x, parent: %x)", in.localID, header.ParentID) + t.Fatalf("parent for proposal not found parent: %x", header.ParentID) } - // set the height and chain ID - header.ChainID = parent.ChainID - header.Height = parent.Height + 1 - // convert into proposal immediately - proposal := model.ProposalFromFlow(header, parent.View) + proposal := model.ProposalFromFlow(header) // store locally and loop back to engine for processing in.ProcessBlock(proposal) - - return nil }, ) - in.communicator.On("SendVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + in.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutObject, ok := args[0].(*model.TimeoutObject) + require.True(t, ok) + in.queue <- timeoutObject + }, + ) + // in case of single node setup we should just forward vote to our own node + // for multi-node setup this method will be overridden + in.notifier.On("OnOwnVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + in.queue <- &model.Vote{ + View: args[1].(uint64), + BlockID: args[0].(flow.Identifier), + SignerID: in.localID, + SigData: args[2].([]byte), + } + }) // program the finalizer module behaviour in.finalizer.On("MakeFinal", mock.Anything).Return( @@ -283,23 +342,14 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { in.builder.Calls = nil in.signer.Calls = nil in.verifier.Calls = nil - in.communicator.Calls = nil + in.notifier.Calls = nil in.finalizer.Calls = nil } - // check on stop condition - // TODO: we can remove that once the single instance stop - // recursively calling into itself - if in.stop(&in) { - return errStopCondition - } - return nil }, ) - in.finalizer.On("MakeValid", mock.Anything).Return(nil) - // initialize error handling and logging var err error zerolog.TimestampFunc = func() time.Time { return time.Now().UTC() } @@ -308,19 +358,17 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { Int("index", int(index)). Hex("node_id", in.localID[:]). Logger() - notifier := notifications.NewLogConsumer(log) - - // initialize the pacemaker - controller := timeout.NewController(cfg.Timeouts) - in.pacemaker, err = pacemaker.New(DefaultStart(), controller, notifier) - require.NoError(t, err) + notifier := pubsub.NewDistributor() + logConsumer := notifications.NewLogConsumer(log) + notifier.AddConsumer(logConsumer) + notifier.AddConsumer(in.notifier) // initialize the block producer in.producer, err = blockproducer.New(in.signer, in.committee, in.builder) require.NoError(t, err) // initialize the finalizer - rootBlock := model.BlockFromFlow(cfg.Root, 0) + rootBlock := model.BlockFromFlow(cfg.Root) signerIndices, err := msig.EncodeSignersToIndices(in.participants.NodeIDs(), in.participants.NodeIDs()) require.NoError(t, err, "could not encode signer indices") @@ -331,25 +379,27 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { SignerIndices: signerIndices, } rootBlockQC := &forks.BlockQC{Block: rootBlock, QC: rootQC} - forkalizer, err := finalizer.New(rootBlockQC, in.finalizer, notifier) - require.NoError(t, err) - // initialize the forks choice - choice, err := forkchoice.NewNewestForkChoice(forkalizer, notifier) + livenessData := &hotstuff.LivenessData{ + CurrentView: rootQC.View + 1, + NewestQC: rootQC, + } + + in.persist.On("GetLivenessData").Return(livenessData, nil).Once() + + // initialize the pacemaker + controller := timeout.NewController(cfg.Timeouts) + in.pacemaker, err = pacemaker.New(controller, notifier, in.persist) require.NoError(t, err) // initialize the forks handler - in.forks = forks.New(forkalizer, choice) + in.forks, err = forks.New(rootBlockQC, in.finalizer, notifier) + require.NoError(t, err) // initialize the validator - in.validator = validator.New(in.committee, in.forks, in.verifier) + in.validator = validator.New(in.committee, in.verifier) weight := uint64(flow.DefaultInitialWeight) - stakingSigAggtor := helper.MakeWeightedSignatureAggregator(weight) - stakingSigAggtor.On("Verify", mock.Anything, mock.Anything).Return(nil).Maybe() - - rbRector := helper.MakeRandomBeaconReconstructor(msig.RandomBeaconThreshold(int(in.participants.Count()))) - rbRector.On("Verify", mock.Anything, mock.Anything).Return(nil).Maybe() indices, err := msig.EncodeSignersToIndices(in.participants.NodeIDs(), []flow.Identifier(in.participants.NodeIDs())) require.NoError(t, err) @@ -361,10 +411,16 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { in.queue <- qc } - minRequiredWeight := hotstuff.ComputeWeightThresholdForBuildingQC(uint64(in.participants.Count()) * weight) - voteProcessorFactory := &mocks.VoteProcessorFactory{} + minRequiredWeight := committees.WeightThresholdToBuildQC(uint64(in.participants.Count()) * weight) + voteProcessorFactory := mocks.NewVoteProcessorFactory(t) voteProcessorFactory.On("Create", mock.Anything, mock.Anything).Return( func(log zerolog.Logger, proposal *model.Proposal) hotstuff.VerifyingVoteProcessor { + stakingSigAggtor := helper.MakeWeightedSignatureAggregator(weight) + stakingSigAggtor.On("Verify", mock.Anything, mock.Anything).Return(nil).Maybe() + + rbRector := helper.MakeRandomBeaconReconstructor(msig.RandomBeaconThreshold(int(in.participants.Count()))) + rbRector.On("Verify", mock.Anything, mock.Anything).Return(nil).Maybe() + return votecollector.NewCombinedVoteProcessor( log, proposal.Block, stakingSigAggtor, rbRector, @@ -372,22 +428,118 @@ func NewInstance(t require.TestingT, options ...Option) *Instance { packer, minRequiredWeight, ) - }, nil) + }, nil).Maybe() createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) - voteCollectors := voteaggregator.NewVoteCollectors(log, DefaultPruned(), workerpool.New(2), createCollectorFactoryMethod) + voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) + + metricsCollector := metrics.NewNoopCollector() // initialize the vote aggregator - in.aggregator, err = voteaggregator.NewVoteAggregator(log, notifier, DefaultPruned(), voteCollectors) + in.voteAggregator, err = voteaggregator.NewVoteAggregator( + log, + metricsCollector, + metricsCollector, + metricsCollector, + notifier, + livenessData.CurrentView, + voteCollectors, + ) require.NoError(t, err) - // initialize the voter - in.voter = voter.New(in.signer, in.forks, in.persist, in.committee, DefaultVoted()) + // initialize factories for timeout collector and timeout processor + collectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutProcessorFactory := mocks.NewTimeoutProcessorFactory(t) + timeoutProcessorFactory.On("Create", mock.Anything).Return( + func(view uint64) hotstuff.TimeoutProcessor { + // mock signature aggregator which doesn't perform any crypto operations and just tracks total weight + aggregator := &mocks.TimeoutSignatureAggregator{} + totalWeight := atomic.NewUint64(0) + newestView := counters.NewMonotonousCounter(0) + aggregator.On("View").Return(view).Maybe() + aggregator.On("TotalWeight").Return(func() uint64 { + return totalWeight.Load() + }).Maybe() + aggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything).Return( + func(signerID flow.Identifier, _ crypto.Signature, newestQCView uint64) uint64 { + newestView.Set(newestQCView) + identity, ok := in.participants.ByNodeID(signerID) + require.True(t, ok) + return totalWeight.Add(identity.Weight) + }, nil, + ).Maybe() + aggregator.On("Aggregate", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return( + func() []hotstuff.TimeoutSignerInfo { + signersData := make([]hotstuff.TimeoutSignerInfo, 0, len(in.participants)) + newestQCView := newestView.Value() + for _, signer := range in.participants.NodeIDs() { + signersData = append(signersData, hotstuff.TimeoutSignerInfo{ + NewestQCView: newestQCView, + Signer: signer, + }) + } + return signersData + }, + unittest.SignatureFixture(), + nil, + ).Maybe() + + p, err := timeoutcollector.NewTimeoutProcessor( + unittest.Logger(), + in.committee, + in.validator, + aggregator, + collectorDistributor, + ) + require.NoError(t, err) + return p + }, nil).Maybe() + timeoutCollectorFactory := timeoutcollector.NewTimeoutCollectorFactory( + unittest.Logger(), + notifier, + collectorDistributor, + timeoutProcessorFactory, + ) + timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorFactory) + + // initialize the timeout aggregator + in.timeoutAggregator, err = timeoutaggregator.NewTimeoutAggregator( + log, + metricsCollector, + metricsCollector, + metricsCollector, + notifier, + livenessData.CurrentView, + timeoutCollectors, + ) + require.NoError(t, err) + + safetyData := &hotstuff.SafetyData{ + LockedOneChainView: rootBlock.View, + HighestAcknowledgedView: rootBlock.View, + } + in.persist.On("GetSafetyData", mock.Anything).Return(safetyData, nil).Once() + + // initialize the safety rules + in.safetyRules, err = safetyrules.New(in.signer, in.persist, in.committee) + require.NoError(t, err) // initialize the event handler - in.handler, err = eventhandler.NewEventHandler(log, in.pacemaker, in.producer, in.forks, in.persist, in.communicator, in.committee, in.aggregator, in.voter, in.validator, notifier) + in.handler, err = eventhandler.NewEventHandler( + log, + in.pacemaker, + in.producer, + in.forks, + in.persist, + in.committee, + in.safetyRules, + notifier, + ) require.NoError(t, err) + collectorDistributor.AddConsumer(logConsumer) + collectorDistributor.AddConsumer(&in) + return &in } @@ -395,14 +547,15 @@ func (in *Instance) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer func() { cancel() - <-in.aggregator.Done() + <-util.AllDone(in.voteAggregator, in.timeoutAggregator) }() signalerCtx, _ := irrecoverable.WithSignaler(ctx) - in.aggregator.Start(signalerCtx) - <-in.aggregator.Ready() + in.voteAggregator.Start(signalerCtx) + in.timeoutAggregator.Start(signalerCtx) + <-util.AllReady(in.voteAggregator, in.timeoutAggregator) // start the event handler - err := in.handler.Start() + err := in.handler.Start(ctx) if err != nil { return fmt.Errorf("could not start event handler: %w", err) } @@ -440,27 +593,41 @@ func (in *Instance) Run() error { case msg := <-in.queue: switch m := msg.(type) { case *model.Proposal: + // add block to aggregator + in.voteAggregator.AddBlock(m) + // then pass to event handler err := in.handler.OnReceiveProposal(m) if err != nil { return fmt.Errorf("could not process proposal: %w", err) } case *model.Vote: - in.aggregator.AddVote(m) + in.voteAggregator.AddVote(m) + case *model.TimeoutObject: + in.timeoutAggregator.AddTimeout(m) case *flow.QuorumCertificate: - err := in.handler.OnQCConstructed(m) + err := in.handler.OnReceiveQc(m) + if err != nil { + return fmt.Errorf("could not process received QC: %w", err) + } + case *flow.TimeoutCertificate: + err := in.handler.OnReceiveTc(m) + if err != nil { + return fmt.Errorf("could not process received TC: %w", err) + } + case *hotstuff.PartialTcCreated: + err := in.handler.OnPartialTcCreated(m) if err != nil { - return fmt.Errorf("could not process created qc: %w", err) + return fmt.Errorf("could not process partial TC: %w", err) } } } - } } func (in *Instance) ProcessBlock(proposal *model.Proposal) { in.updatingBlocks.Lock() - _, parentExists := in.headers[proposal.Block.QC.BlockID] defer in.updatingBlocks.Unlock() + _, parentExists := in.headers[proposal.Block.QC.BlockID] if parentExists { next := proposal @@ -476,3 +643,23 @@ func (in *Instance) ProcessBlock(proposal *model.Proposal) { in.pendings[proposal.Block.QC.BlockID] = proposal } } + +func (in *Instance) OnTcConstructedFromTimeouts(tc *flow.TimeoutCertificate) { + in.queue <- tc +} + +func (in *Instance) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + in.queue <- &hotstuff.PartialTcCreated{ + View: view, + NewestQC: newestQC, + LastViewTC: lastViewTC, + } +} + +func (in *Instance) OnNewQcDiscovered(qc *flow.QuorumCertificate) { + in.queue <- qc +} + +func (in *Instance) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { + in.queue <- tc +} diff --git a/consensus/hotstuff/integration/integration_test.go b/consensus/hotstuff/integration/integration_test.go index 384c542af91..e2929777dee 100644 --- a/consensus/hotstuff/integration/integration_test.go +++ b/consensus/hotstuff/integration/integration_test.go @@ -17,7 +17,8 @@ import ( // but for slow environment like CI, a longer one is needed. const safeTimeout = 2 * time.Second -const safeDecreaseFactor = 0.85 +// number of failed rounds before first timeout increase +const happyPathMaxRoundFailures = 6 func TestSingleInstance(t *testing.T) { @@ -51,10 +52,12 @@ func TestThreeInstances(t *testing.T) { // generate three hotstuff participants participants := unittest.IdentityListFixture(num) root := DefaultRoot() - timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 0.5, 1.5, safeDecreaseFactor, 0) + timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, 0, safeTimeout) require.NoError(t, err) // set up three instances that are exactly the same + // since we don't block any messages we should have enough data to advance in happy path + // for that reason we will block all TO related communication. instances := make([]*Instance, 0, num) for n := 0; n < num; n++ { in := NewInstance(t, @@ -63,12 +66,13 @@ func TestThreeInstances(t *testing.T) { WithLocalID(participants[n].NodeID), WithTimeouts(timeouts), WithStopCondition(ViewFinalized(finalView)), + WithIncomingTimeoutObjects(BlockAllTimeoutObjects), ) instances = append(instances, in) } // connect the communicators of the instances together - Connect(instances) + Connect(t, instances) // start the instances and wait for them to finish var wg sync.WaitGroup @@ -112,7 +116,7 @@ func TestSevenInstances(t *testing.T) { participants := unittest.IdentityListFixture(numPass + numFail) instances := make([]*Instance, 0, numPass+numFail) root := DefaultRoot() - timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 0.5, 1.5, safeDecreaseFactor, 0) + timeouts, err := timeout.NewConfig(safeTimeout, safeTimeout, 1.5, happyPathMaxRoundFailures, 0, safeTimeout) require.NoError(t, err) // set up five instances that work fully @@ -141,7 +145,7 @@ func TestSevenInstances(t *testing.T) { } // connect the communicators of the instances together - Connect(instances) + Connect(t, instances) // start all seven instances and wait for them to wrap up var wg sync.WaitGroup diff --git a/consensus/hotstuff/integration/liveness_test.go b/consensus/hotstuff/integration/liveness_test.go new file mode 100644 index 00000000000..b9eca3cf005 --- /dev/null +++ b/consensus/hotstuff/integration/liveness_test.go @@ -0,0 +1,408 @@ +package integration + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// pacemaker timeout +// if your laptop is fast enough, 10 ms is enough +const pmTimeout = 100 * time.Millisecond + +// maxTimeoutRebroadcast specifies how often the PaceMaker rebroadcasts +// its timeout object in case there is no progress. We keep the value +// small so we have smaller latency +const maxTimeoutRebroadcast = 1 * time.Second + +// If 2 nodes are down in a 7 nodes cluster, the rest of 5 nodes can +// still make progress and reach consensus +func Test2TimeoutOutof7Instances(t *testing.T) { + + healthyReplicas := 5 + notVotingReplicas := 2 + finalView := uint64(30) + + // generate the seven hotstuff participants + participants := unittest.IdentityListFixture(healthyReplicas + notVotingReplicas) + instances := make([]*Instance, 0, healthyReplicas+notVotingReplicas) + root := DefaultRoot() + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + require.NoError(t, err) + + // set up five instances that work fully + for n := 0; n < healthyReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + ) + instances = append(instances, in) + } + + // set up two instances which can't vote, nor propose + for n := healthyReplicas; n < healthyReplicas+notVotingReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + WithOutgoingVotes(BlockAllVotes), + WithOutgoingProposals(BlockAllProposals), + ) + instances = append(instances, in) + } + + // connect the communicators of the instances together + Connect(t, instances) + + // start all seven instances and wait for them to wrap up + var wg sync.WaitGroup + for _, in := range instances { + wg.Add(1) + go func(in *Instance) { + err := in.Run() + require.True(t, errors.Is(err, errStopCondition)) + wg.Done() + }(in) + } + unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second, "expect to finish before timeout") + + // check that all instances have the same finalized block + ref := instances[0] + assert.Equal(t, finalView, ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") + finalizedViews := FinalizedViews(ref) + for i := 1; i < healthyReplicas; i++ { + assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") + assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") + } +} + +// 2 nodes in a 4-node cluster are configured to be able only to send timeout messages (no voting or proposing). +// The other 2 unconstrained nodes should be able to make progress through the recovery path by creating TCs +// for every round, but no block will be finalized, because finalization requires direct 1-chain and QC. +func Test2TimeoutOutof4Instances(t *testing.T) { + + healthyReplicas := 2 + replicasDroppingHappyPathMsgs := 2 + finalView := uint64(30) + + // generate the 4 hotstuff participants + participants := unittest.IdentityListFixture(healthyReplicas + replicasDroppingHappyPathMsgs) + instances := make([]*Instance, 0, healthyReplicas+replicasDroppingHappyPathMsgs) + root := DefaultRoot() + timeouts, err := timeout.NewConfig( + 10*time.Millisecond, 50*time.Millisecond, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + require.NoError(t, err) + + // set up two instances that work fully + for n := 0; n < healthyReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewReached(finalView)), + ) + instances = append(instances, in) + } + + // set up instances which can't vote, nor propose + for n := healthyReplicas; n < healthyReplicas+replicasDroppingHappyPathMsgs; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewReached(finalView)), + WithOutgoingVotes(BlockAllVotes), + WithIncomingVotes(BlockAllVotes), + WithOutgoingProposals(BlockAllProposals), + ) + instances = append(instances, in) + } + + // connect the communicators of the instances together + Connect(t, instances) + + // start the instances and wait for them to finish + var wg sync.WaitGroup + for _, in := range instances { + wg.Add(1) + go func(in *Instance) { + err := in.Run() + require.True(t, errors.Is(err, errStopCondition), "should run until stop condition") + wg.Done() + }(in) + } + unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second, "expect to finish before timeout") + + // check that all instances have the same finalized block + ref := instances[0] + finalizedViews := FinalizedViews(ref) + assert.Equal(t, []uint64{0}, finalizedViews, "no view was finalized, because finalization requires 2 direct chain plus a QC which never happen in this case") + assert.Equal(t, finalView, ref.pacemaker.CurView(), "expect instance 0 should made enough progress, but didn't") + for i := 1; i < healthyReplicas; i++ { + assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance", i) + assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance", i) + assert.Equal(t, finalView, instances[i].pacemaker.CurView(), "instance %d should have same active view as first instance", i) + } +} + +// If 1 node is down in a 5 nodes cluster, the rest of 4 nodes can +// make progress and reach consensus +func Test1TimeoutOutof5Instances(t *testing.T) { + + healthyReplicas := 4 + blockedReplicas := 1 + finalView := uint64(30) + + // generate the seven hotstuff participants + participants := unittest.IdentityListFixture(healthyReplicas + blockedReplicas) + instances := make([]*Instance, 0, healthyReplicas+blockedReplicas) + root := DefaultRoot() + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, 0, maxTimeoutRebroadcast) + require.NoError(t, err) + + // set up instances that work fully + for n := 0; n < healthyReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + ) + instances = append(instances, in) + } + + // set up one instance which can't vote, nor propose + for n := healthyReplicas; n < healthyReplicas+blockedReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewReached(finalView)), + WithOutgoingVotes(BlockAllVotes), + WithOutgoingProposals(BlockAllProposals), + ) + instances = append(instances, in) + } + + // connect the communicators of the instances together + Connect(t, instances) + + // start all seven instances and wait for them to wrap up + var wg sync.WaitGroup + for _, in := range instances { + wg.Add(1) + go func(in *Instance) { + err := in.Run() + require.True(t, errors.Is(err, errStopCondition)) + wg.Done() + }(in) + } + success := unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second, "expect to finish before timeout") + if !success { + t.Logf("dumping state of system:") + for i, inst := range instances { + t.Logf( + "instance %d: %d %d %d %d", + i, + inst.pacemaker.CurView(), + inst.pacemaker.NewestQC().View, + inst.forks.FinalizedBlock().View, + inst.forks.NewestView(), + ) + } + } + + // check that all instances have the same finalized block + ref := instances[0] + finalizedViews := FinalizedViews(ref) + assert.Equal(t, finalView, ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") + for i := 1; i < healthyReplicas; i++ { + assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") + assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") + } +} + +// TestBlockDelayIsHigherThanTimeout tests an edge case protocol edge case, where +// - The block arrives in time for replicas to vote. +// - The next primary does not respond in time with a follow-up proposal, +// so nodes start sending TimeoutObjects. +// - However, eventually, the next primary successfully constructs a QC and a new +// block before a TC leads to the round timing out. +// +// This test verifies that nodes still make progress on the happy path (QC constructed), +// despite already having initiated the timeout. +// Example scenarios, how this timing edge case could manifest: +// - block delay is very close (or larger) than round duration +// - delayed message transmission (specifically votes) within network +// - overwhelmed / slowed-down primary +// - byzantine primary +// +// Implementation: +// - We have 4 nodes in total where the TimeoutObjects from two of them are always +// discarded. Therefore, no TC can be constructed. +// - To force nodes to initiate the timeout (i.e. send TimeoutObjects), we set +// the `blockRateDelay` to _twice_ the PaceMaker Timeout. Furthermore, we configure +// the PaceMaker to only increase timeout duration after 6 successive round failures. +func TestBlockDelayIsHigherThanTimeout(t *testing.T) { + healthyReplicas := 2 + replicasNotGeneratingTimeouts := 2 + finalView := uint64(20) + + // generate the 4 hotstuff participants + participants := unittest.IdentityListFixture(healthyReplicas + replicasNotGeneratingTimeouts) + instances := make([]*Instance, 0, healthyReplicas+replicasNotGeneratingTimeouts) + root := DefaultRoot() + // set block rate delay to be bigger than minimal timeout + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, happyPathMaxRoundFailures, pmTimeout*2, maxTimeoutRebroadcast) + require.NoError(t, err) + + // set up 2 instances that fully work (incl. sending TimeoutObjects) + for n := 0; n < healthyReplicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + ) + instances = append(instances, in) + } + + // set up two instances which don't generate and receive timeout objects + for n := healthyReplicas; n < healthyReplicas+replicasNotGeneratingTimeouts; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + WithIncomingTimeoutObjects(BlockAllTimeoutObjects), + WithOutgoingTimeoutObjects(BlockAllTimeoutObjects), + ) + instances = append(instances, in) + } + + // connect the communicators of the instances together + Connect(t, instances) + + // start all 4 instances and wait for them to wrap up + var wg sync.WaitGroup + for _, in := range instances { + wg.Add(1) + go func(in *Instance) { + err := in.Run() + require.True(t, errors.Is(err, errStopCondition)) + wg.Done() + }(in) + } + unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second, "expect to finish before timeout") + + // check that all instances have the same finalized block + ref := instances[0] + assert.Equal(t, finalView, ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") + finalizedViews := FinalizedViews(ref) + // in this test we rely on QC being produced in each view + // make sure that all views are strictly in increasing order with no gaps + for i := 1; i < len(finalizedViews); i++ { + // finalized views are sorted in descending order + if finalizedViews[i-1] != finalizedViews[i]+1 { + t.Fatalf("finalized views series has gap, this is not expected: %v", finalizedViews) + return + } + } + for i := 1; i < healthyReplicas; i++ { + assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") + assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") + } +} + +// TestAsyncClusterStartup tests a realistic scenario where nodes are started asynchronously: +// - Replicas are started in sequential order +// - Each replica skips voting for first block(emulating message omission). +// - Each replica skips first Timeout Object [TO] (emulating message omission). +// - At this point protocol loses liveness unless a timeout rebroadcast happens from super-majority of replicas. +// +// This test verifies that nodes still make progress, despite first TO messages being lost. +// Implementation: +// - We have 4 replicas in total, each of them skips voting for first view to force a timeout +// - Block TOs for whole committee until each replica has generated its first TO. +// - After each replica has generated a timeout allow subsequent timeout rebroadcasts to make progress. +func TestAsyncClusterStartup(t *testing.T) { + replicas := 4 + finalView := uint64(20) + + // generate the seven hotstuff participants + participants := unittest.IdentityListFixture(replicas) + instances := make([]*Instance, 0, replicas) + root := DefaultRoot() + // set block rate delay to be bigger than minimal timeout + timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 1.5, 6, 0, maxTimeoutRebroadcast) + require.NoError(t, err) + + // set up instances that work fully + var lock sync.Mutex + timeoutObjectGenerated := make(map[flow.Identifier]struct{}, 0) + for n := 0; n < replicas; n++ { + in := NewInstance(t, + WithRoot(root), + WithParticipants(participants), + WithLocalID(participants[n].NodeID), + WithTimeouts(timeouts), + WithStopCondition(ViewFinalized(finalView)), + WithOutgoingVotes(func(vote *model.Vote) bool { + return vote.View == 1 + }), + WithOutgoingTimeoutObjects(func(object *model.TimeoutObject) bool { + lock.Lock() + defer lock.Unlock() + timeoutObjectGenerated[object.SignerID] = struct{}{} + // start allowing timeouts when every node has generated one + // when nodes will broadcast again, it will go through + return len(timeoutObjectGenerated) != replicas + }), + ) + instances = append(instances, in) + } + + // connect the communicators of the instances together + Connect(t, instances) + + // start each node only after previous one has started + var wg sync.WaitGroup + for _, in := range instances { + wg.Add(1) + go func(in *Instance) { + err := in.Run() + require.True(t, errors.Is(err, errStopCondition)) + wg.Done() + }(in) + } + unittest.AssertReturnsBefore(t, wg.Wait, 10*time.Second, "expect to finish before timeout") + + // check that all instances have the same finalized block + ref := instances[0] + assert.Equal(t, finalView, ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") + finalizedViews := FinalizedViews(ref) + for i := 1; i < replicas; i++ { + assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") + assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") + } +} diff --git a/consensus/hotstuff/integration/options_test.go b/consensus/hotstuff/integration/options_test.go index 749387b07f6..5a498579d05 100644 --- a/consensus/hotstuff/integration/options_test.go +++ b/consensus/hotstuff/integration/options_test.go @@ -12,15 +12,18 @@ var errStopCondition = errors.New("stop condition reached") type Option func(*Config) type Config struct { - Root *flow.Header - Participants flow.IdentityList - LocalID flow.Identifier - Timeouts timeout.Config - IncomingVotes VoteFilter - OutgoingVotes VoteFilter - IncomingProposals ProposalFilter - OutgoingProposals ProposalFilter - StopCondition Condition + Root *flow.Header + Participants flow.IdentityList + LocalID flow.Identifier + Timeouts timeout.Config + IncomingVotes VoteFilter + OutgoingVotes VoteFilter + IncomingTimeoutObjects TimeoutObjectFilter + OutgoingTimeoutObjects TimeoutObjectFilter + IncomingProposals ProposalFilter + OutgoingProposals ProposalFilter + + StopCondition Condition } func WithRoot(root *flow.Header) Option { @@ -71,6 +74,18 @@ func WithOutgoingProposals(Filter ProposalFilter) Option { } } +func WithIncomingTimeoutObjects(Filter TimeoutObjectFilter) Option { + return func(cfg *Config) { + cfg.IncomingTimeoutObjects = Filter + } +} + +func WithOutgoingTimeoutObjects(Filter TimeoutObjectFilter) Option { + return func(cfg *Config) { + cfg.OutgoingTimeoutObjects = Filter + } +} + func WithStopCondition(stop Condition) Option { return func(cfg *Config) { cfg.StopCondition = stop diff --git a/consensus/hotstuff/integration/slow_test.go b/consensus/hotstuff/integration/slow_test.go deleted file mode 100644 index af4554df3a1..00000000000 --- a/consensus/hotstuff/integration/slow_test.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build timesensitivetest -// +build timesensitivetest - -// This file includes a few time sensitive tests. They might pass on your powerful local machine -// but fail on slow CI machine. -// For now, these tests are only for local. Run it with the build tag on: -// > go test --tags=relic,timesensitivetest ./... - -package integration - -import ( - "errors" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" - "github.com/onflow/flow-go/utils/unittest" -) - -// pacemaker timeout -// if your laptop is fast enough, 10 ms is enough -const pmTimeout = 10 * time.Millisecond - -// If 2 nodes are down in a 7 nodes cluster, the rest of 5 nodes can -// still make progress and reach consensus -func Test2TimeoutOutof7Instances(t *testing.T) { - - // test parameters - // NOTE: block finalization seems to be rather slow on CI at the moment, - // needing around 1 minute on Travis for 1000 blocks and 10 minutes on - // TeamCity for 1000 blocks; in order to avoid test timeouts, we keep the - // number low here - numPass := 5 - numFail := 2 - finalView := uint64(30) - - // generate the seven hotstuff participants - participants := unittest.IdentityListFixture(numPass + numFail) - instances := make([]*Instance, 0, numPass+numFail) - root := DefaultRoot() - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 0.5, 1.5, 0.85, 0) - require.NoError(t, err) - - // set up five instances that work fully - for n := 0; n < numPass; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - ) - instances = append(instances, in) - } - - // set up two instances which can't vote, nor propose - for n := numPass; n < numPass+numFail; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - WithOutgoingVotes(BlockAllVotes), - WithOutgoingProposals(BlockAllProposals), - WithIncomingProposals(BlockAllProposals), - ) - instances = append(instances, in) - } - - // connect the communicators of the instances together - Connect(instances) - - // start all seven instances and wait for them to wrap up - var wg sync.WaitGroup - for _, in := range instances { - wg.Add(1) - go func(in *Instance) { - err := in.Run() - require.True(t, errors.Is(err, errStopCondition)) - wg.Done() - }(in) - } - wg.Wait() - - // check that all instances have the same finalized block - ref := instances[0] - assert.Less(t, finalView-uint64(2*numPass+numFail), ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") - finalizedViews := FinalizedViews(ref) - for i := 1; i < numPass; i++ { - assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") - assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") - } -} - -// If 1 node is down in a 4 nodes cluster, the rest of 3 nodes can -// still make progress, but no block will be finalized, because -// finalization requires 2-direct chain and a QC -func Test1TimeoutOutof4Instances(t *testing.T) { - - // test parameters - // NOTE: block finalization seems to be rather slow on CI at the moment, - // needing around 1 minute on Travis for 1000 blocks and 10 minutes on - // TeamCity for 1000 blocks; in order to avoid test timeouts, we keep the - // number low here - numPass := 3 - numFail := 1 - finalView := uint64(30) - - // generate the 4 hotstuff participants - participants := unittest.IdentityListFixture(numPass + numFail) - instances := make([]*Instance, 0, numPass+numFail) - root := DefaultRoot() - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 0.5, 1.5, 0.85, 0) - require.NoError(t, err) - - // set up three instances that work fully - for n := 0; n < numPass; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - ) - instances = append(instances, in) - } - - // set up one instance which can't vote, nor propose - for n := numPass; n < numPass+numFail; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - WithOutgoingVotes(BlockAllVotes), - WithOutgoingProposals(BlockAllProposals), - WithIncomingProposals(BlockAllProposals), - ) - instances = append(instances, in) - } - - // connect the communicators of the instances together - Connect(instances) - - // start the instances and wait for them to finish - var wg sync.WaitGroup - for _, in := range instances { - wg.Add(1) - go func(in *Instance) { - err := in.Run() - require.True(t, errors.Is(err, errStopCondition), "should run until stop condition") - wg.Done() - }(in) - } - wg.Wait() - - // check that all instances have the same finalized block - ref := instances[0] - finalizedViews := FinalizedViews(ref) - assert.Equal(t, []uint64{0}, finalizedViews, "no view was finalized, because finalization requires 2 direct chain plus a QC which never happen in this case") - for i := 1; i < numPass; i++ { - assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") - assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") - } -} - -// If 1 node is down in a 5 nodes cluster, the rest of 4 nodes can -// make progress and reach consensus -func Test1TimeoutOutof5Instances(t *testing.T) { - - // test parameters - // NOTE: block finalization seems to be rather slow on CI at the moment, - // needing around 1 minute on Travis for 1000 blocks and 10 minutes on - // TeamCity for 1000 blocks; in order to avoid test timeouts, we keep the - // number low here - numPass := 4 - numFail := 1 - finalView := uint64(30) - - // generate the seven hotstuff participants - participants := unittest.IdentityListFixture(numPass + numFail) - instances := make([]*Instance, 0, numPass+numFail) - root := DefaultRoot() - timeouts, err := timeout.NewConfig(pmTimeout, pmTimeout, 0.5, 1.5, 0.85, 0) - require.NoError(t, err) - - // set up instances that work fully - for n := 0; n < numPass; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - ) - instances = append(instances, in) - } - - // set up one instance which can't vote, nor propose - for n := numPass; n < numPass+numFail; n++ { - in := NewInstance(t, - WithRoot(root), - WithParticipants(participants), - WithLocalID(participants[n].NodeID), - WithTimeouts(timeouts), - WithStopCondition(ViewReached(finalView)), - WithOutgoingVotes(BlockAllVotes), - WithOutgoingProposals(BlockAllProposals), - WithIncomingProposals(BlockAllProposals), - ) - instances = append(instances, in) - } - - // connect the communicators of the instances together - Connect(instances) - - // start all seven instances and wait for them to wrap up - var wg sync.WaitGroup - for _, in := range instances { - wg.Add(1) - go func(in *Instance) { - err := in.Run() - require.True(t, errors.Is(err, errStopCondition)) - wg.Done() - }(in) - } - wg.Wait() - - // check that all instances have the same finalized block - ref := instances[0] - finalizedViews := FinalizedViews(ref) - assert.Less(t, finalView-uint64(2*numPass+numFail), ref.forks.FinalizedBlock().View, "expect instance 0 should made enough progress, but didn't") - for i := 1; i < numPass; i++ { - assert.Equal(t, ref.forks.FinalizedBlock(), instances[i].forks.FinalizedBlock(), "instance %d should have same finalized block as first instance") - assert.Equal(t, finalizedViews, FinalizedViews(instances[i]), "instance %d should have same finalized view as first instance") - } -} diff --git a/consensus/hotstuff/mocks/block_producer.go b/consensus/hotstuff/mocks/block_producer.go index ac0f6f423bb..1994ad22214 100644 --- a/consensus/hotstuff/mocks/block_producer.go +++ b/consensus/hotstuff/mocks/block_producer.go @@ -6,8 +6,6 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" - - model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // BlockProducer is an autogenerated mock type for the BlockProducer type @@ -15,22 +13,22 @@ type BlockProducer struct { mock.Mock } -// MakeBlockProposal provides a mock function with given fields: qc, view -func (_m *BlockProducer) MakeBlockProposal(qc *flow.QuorumCertificate, view uint64) (*model.Proposal, error) { - ret := _m.Called(qc, view) +// MakeBlockProposal provides a mock function with given fields: view, qc, lastViewTC +func (_m *BlockProducer) MakeBlockProposal(view uint64, qc *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*flow.Header, error) { + ret := _m.Called(view, qc, lastViewTC) - var r0 *model.Proposal - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate, uint64) *model.Proposal); ok { - r0 = rf(qc, view) + var r0 *flow.Header + if rf, ok := ret.Get(0).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) *flow.Header); ok { + r0 = rf(view, qc, lastViewTC) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Proposal) + r0 = ret.Get(0).(*flow.Header) } } var r1 error - if rf, ok := ret.Get(1).(func(*flow.QuorumCertificate, uint64) error); ok { - r1 = rf(qc, view) + if rf, ok := ret.Get(1).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) error); ok { + r1 = rf(view, qc, lastViewTC) } else { r1 = ret.Error(1) } diff --git a/consensus/hotstuff/mocks/communicator.go b/consensus/hotstuff/mocks/communicator.go deleted file mode 100644 index 58787dcfaa1..00000000000 --- a/consensus/hotstuff/mocks/communicator.go +++ /dev/null @@ -1,73 +0,0 @@ -// Code generated by mockery v2.13.1. DO NOT EDIT. - -package mocks - -import ( - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" - - time "time" -) - -// Communicator is an autogenerated mock type for the Communicator type -type Communicator struct { - mock.Mock -} - -// BroadcastProposal provides a mock function with given fields: proposal -func (_m *Communicator) BroadcastProposal(proposal *flow.Header) error { - ret := _m.Called(proposal) - - var r0 error - if rf, ok := ret.Get(0).(func(*flow.Header) error); ok { - r0 = rf(proposal) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// BroadcastProposalWithDelay provides a mock function with given fields: proposal, delay -func (_m *Communicator) BroadcastProposalWithDelay(proposal *flow.Header, delay time.Duration) error { - ret := _m.Called(proposal, delay) - - var r0 error - if rf, ok := ret.Get(0).(func(*flow.Header, time.Duration) error); ok { - r0 = rf(proposal, delay) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// SendVote provides a mock function with given fields: blockID, view, sigData, recipientID -func (_m *Communicator) SendVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) error { - ret := _m.Called(blockID, view, sigData, recipientID) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier, uint64, []byte, flow.Identifier) error); ok { - r0 = rf(blockID, view, sigData, recipientID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewCommunicator interface { - mock.TestingT - Cleanup(func()) -} - -// NewCommunicator creates a new instance of Communicator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewCommunicator(t mockConstructorTestingTNewCommunicator) *Communicator { - mock := &Communicator{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/consensus/hotstuff/mocks/communicator_consumer.go b/consensus/hotstuff/mocks/communicator_consumer.go new file mode 100644 index 00000000000..078602eee72 --- /dev/null +++ b/consensus/hotstuff/mocks/communicator_consumer.go @@ -0,0 +1,48 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" + + time "time" +) + +// CommunicatorConsumer is an autogenerated mock type for the CommunicatorConsumer type +type CommunicatorConsumer struct { + mock.Mock +} + +// OnOwnProposal provides a mock function with given fields: proposal, targetPublicationTime +func (_m *CommunicatorConsumer) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + _m.Called(proposal, targetPublicationTime) +} + +// OnOwnTimeout provides a mock function with given fields: timeout +func (_m *CommunicatorConsumer) OnOwnTimeout(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +// OnOwnVote provides a mock function with given fields: blockID, view, sigData, recipientID +func (_m *CommunicatorConsumer) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + _m.Called(blockID, view, sigData, recipientID) +} + +type mockConstructorTestingTNewCommunicatorConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewCommunicatorConsumer creates a new instance of CommunicatorConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCommunicatorConsumer(t mockConstructorTestingTNewCommunicatorConsumer) *CommunicatorConsumer { + mock := &CommunicatorConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/consumer.go b/consensus/hotstuff/mocks/consumer.go index 03cdbf140ec..372242f3659 100644 --- a/consensus/hotstuff/mocks/consumer.go +++ b/consensus/hotstuff/mocks/consumer.go @@ -3,11 +3,14 @@ package mocks import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" model "github.com/onflow/flow-go/consensus/hotstuff/model" + + time "time" ) // Consumer is an autogenerated mock type for the Consumer type @@ -20,19 +23,24 @@ func (_m *Consumer) OnBlockIncorporated(_a0 *model.Block) { _m.Called(_a0) } +// OnCurrentViewDetails provides a mock function with given fields: currentView, finalizedView, currentLeader +func (_m *Consumer) OnCurrentViewDetails(currentView uint64, finalizedView uint64, currentLeader flow.Identifier) { + _m.Called(currentView, finalizedView, currentLeader) +} + // OnDoubleProposeDetected provides a mock function with given fields: _a0, _a1 func (_m *Consumer) OnDoubleProposeDetected(_a0 *model.Block, _a1 *model.Block) { _m.Called(_a0, _a1) } -// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { +// OnDoubleTimeoutDetected provides a mock function with given fields: _a0, _a1 +func (_m *Consumer) OnDoubleTimeoutDetected(_a0 *model.TimeoutObject, _a1 *model.TimeoutObject) { _m.Called(_a0, _a1) } -// OnEnteringView provides a mock function with given fields: viewNumber, leader -func (_m *Consumer) OnEnteringView(viewNumber uint64, leader flow.Identifier) { - _m.Called(viewNumber, leader) +// OnDoubleVotingDetected provides a mock function with given fields: _a0, _a1 +func (_m *Consumer) OnDoubleVotingDetected(_a0 *model.Vote, _a1 *model.Vote) { + _m.Called(_a0, _a1) } // OnEventProcessed provides a mock function with given fields: @@ -45,39 +53,44 @@ func (_m *Consumer) OnFinalizedBlock(_a0 *model.Block) { _m.Called(_a0) } -// OnForkChoiceGenerated provides a mock function with given fields: _a0, _a1 -func (_m *Consumer) OnForkChoiceGenerated(_a0 uint64, _a1 *flow.QuorumCertificate) { - _m.Called(_a0, _a1) +// OnInvalidTimeoutDetected provides a mock function with given fields: err +func (_m *Consumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + _m.Called(err) } -// OnInvalidVoteDetected provides a mock function with given fields: _a0 -func (_m *Consumer) OnInvalidVoteDetected(_a0 *model.Vote) { - _m.Called(_a0) +// OnInvalidVoteDetected provides a mock function with given fields: err +func (_m *Consumer) OnInvalidVoteDetected(err model.InvalidVoteError) { + _m.Called(err) } -// OnProposingBlock provides a mock function with given fields: proposal -func (_m *Consumer) OnProposingBlock(proposal *model.Proposal) { - _m.Called(proposal) +// OnLocalTimeout provides a mock function with given fields: currentView +func (_m *Consumer) OnLocalTimeout(currentView uint64) { + _m.Called(currentView) } -// OnQcConstructedFromVotes provides a mock function with given fields: curView, qc -func (_m *Consumer) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) { - _m.Called(curView, qc) +// OnOwnProposal provides a mock function with given fields: proposal, targetPublicationTime +func (_m *Consumer) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + _m.Called(proposal, targetPublicationTime) } -// OnQcIncorporated provides a mock function with given fields: _a0 -func (_m *Consumer) OnQcIncorporated(_a0 *flow.QuorumCertificate) { - _m.Called(_a0) +// OnOwnTimeout provides a mock function with given fields: timeout +func (_m *Consumer) OnOwnTimeout(timeout *model.TimeoutObject) { + _m.Called(timeout) } -// OnQcTriggeredViewChange provides a mock function with given fields: qc, newView -func (_m *Consumer) OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) { - _m.Called(qc, newView) +// OnOwnVote provides a mock function with given fields: blockID, view, sigData, recipientID +func (_m *Consumer) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + _m.Called(blockID, view, sigData, recipientID) } -// OnReachedTimeout provides a mock function with given fields: timeout -func (_m *Consumer) OnReachedTimeout(timeout *model.TimerInfo) { - _m.Called(timeout) +// OnPartialTc provides a mock function with given fields: currentView, partialTc +func (_m *Consumer) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + _m.Called(currentView, partialTc) +} + +// OnQcTriggeredViewChange provides a mock function with given fields: oldView, newView, qc +func (_m *Consumer) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + _m.Called(oldView, newView, qc) } // OnReceiveProposal provides a mock function with given fields: currentView, proposal @@ -85,23 +98,48 @@ func (_m *Consumer) OnReceiveProposal(currentView uint64, proposal *model.Propos _m.Called(currentView, proposal) } -// OnReceiveVote provides a mock function with given fields: currentView, vote -func (_m *Consumer) OnReceiveVote(currentView uint64, vote *model.Vote) { - _m.Called(currentView, vote) +// OnReceiveQc provides a mock function with given fields: currentView, qc +func (_m *Consumer) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + _m.Called(currentView, qc) +} + +// OnReceiveTc provides a mock function with given fields: currentView, tc +func (_m *Consumer) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + _m.Called(currentView, tc) +} + +// OnStart provides a mock function with given fields: currentView +func (_m *Consumer) OnStart(currentView uint64) { + _m.Called(currentView) } // OnStartingTimeout provides a mock function with given fields: _a0 -func (_m *Consumer) OnStartingTimeout(_a0 *model.TimerInfo) { +func (_m *Consumer) OnStartingTimeout(_a0 model.TimerInfo) { _m.Called(_a0) } +// OnTcTriggeredViewChange provides a mock function with given fields: oldView, newView, tc +func (_m *Consumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + _m.Called(oldView, newView, tc) +} + +// OnTimeoutProcessed provides a mock function with given fields: timeout +func (_m *Consumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + _m.Called(timeout) +} + +// OnViewChange provides a mock function with given fields: oldView, newView +func (_m *Consumer) OnViewChange(oldView uint64, newView uint64) { + _m.Called(oldView, newView) +} + // OnVoteForInvalidBlockDetected provides a mock function with given fields: vote, invalidProposal func (_m *Consumer) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidProposal *model.Proposal) { _m.Called(vote, invalidProposal) } -// OnVoting provides a mock function with given fields: vote -func (_m *Consumer) OnVoting(vote *model.Vote) { +// OnVoteProcessed provides a mock function with given fields: vote +func (_m *Consumer) OnVoteProcessed(vote *model.Vote) { _m.Called(vote) } diff --git a/consensus/hotstuff/mocks/dynamic_committee.go b/consensus/hotstuff/mocks/dynamic_committee.go new file mode 100644 index 00000000000..b3ce6287327 --- /dev/null +++ b/consensus/hotstuff/mocks/dynamic_committee.go @@ -0,0 +1,226 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" +) + +// DynamicCommittee is an autogenerated mock type for the DynamicCommittee type +type DynamicCommittee struct { + mock.Mock +} + +// DKG provides a mock function with given fields: view +func (_m *DynamicCommittee) DKG(view uint64) (hotstuff.DKG, error) { + ret := _m.Called(view) + + var r0 hotstuff.DKG + if rf, ok := ret.Get(0).(func(uint64) hotstuff.DKG); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(hotstuff.DKG) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentitiesByBlock provides a mock function with given fields: blockID +func (_m *DynamicCommittee) IdentitiesByBlock(blockID flow.Identifier) (flow.IdentityList, error) { + ret := _m.Called(blockID) + + var r0 flow.IdentityList + if rf, ok := ret.Get(0).(func(flow.Identifier) flow.IdentityList); ok { + r0 = rf(blockID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.IdentityList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(blockID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentitiesByEpoch provides a mock function with given fields: view +func (_m *DynamicCommittee) IdentitiesByEpoch(view uint64) (flow.IdentityList, error) { + ret := _m.Called(view) + + var r0 flow.IdentityList + if rf, ok := ret.Get(0).(func(uint64) flow.IdentityList); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.IdentityList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentityByBlock provides a mock function with given fields: blockID, participantID +func (_m *DynamicCommittee) IdentityByBlock(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { + ret := _m.Called(blockID, participantID) + + var r0 *flow.Identity + if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) *flow.Identity); ok { + r0 = rf(blockID, participantID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Identity) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(flow.Identifier, flow.Identifier) error); ok { + r1 = rf(blockID, participantID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentityByEpoch provides a mock function with given fields: view, participantID +func (_m *DynamicCommittee) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.Identity, error) { + ret := _m.Called(view, participantID) + + var r0 *flow.Identity + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) *flow.Identity); ok { + r0 = rf(view, participantID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Identity) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64, flow.Identifier) error); ok { + r1 = rf(view, participantID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LeaderForView provides a mock function with given fields: view +func (_m *DynamicCommittee) LeaderForView(view uint64) (flow.Identifier, error) { + ret := _m.Called(view) + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func(uint64) flow.Identifier); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// QuorumThresholdForView provides a mock function with given fields: view +func (_m *DynamicCommittee) QuorumThresholdForView(view uint64) (uint64, error) { + ret := _m.Called(view) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(view) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Self provides a mock function with given fields: +func (_m *DynamicCommittee) Self() flow.Identifier { + ret := _m.Called() + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func() flow.Identifier); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + return r0 +} + +// TimeoutThresholdForView provides a mock function with given fields: view +func (_m *DynamicCommittee) TimeoutThresholdForView(view uint64) (uint64, error) { + ret := _m.Called(view) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(view) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewDynamicCommittee interface { + mock.TestingT + Cleanup(func()) +} + +// NewDynamicCommittee creates a new instance of DynamicCommittee. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewDynamicCommittee(t mockConstructorTestingTNewDynamicCommittee) *DynamicCommittee { + mock := &DynamicCommittee{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/event_handler.go b/consensus/hotstuff/mocks/event_handler.go index 7c6e344a96b..8cafa8b06c6 100644 --- a/consensus/hotstuff/mocks/event_handler.go +++ b/consensus/hotstuff/mocks/event_handler.go @@ -3,6 +3,9 @@ package mocks import ( + context "context" + + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" @@ -31,13 +34,13 @@ func (_m *EventHandler) OnLocalTimeout() error { return r0 } -// OnQCConstructed provides a mock function with given fields: qc -func (_m *EventHandler) OnQCConstructed(qc *flow.QuorumCertificate) error { - ret := _m.Called(qc) +// OnPartialTcCreated provides a mock function with given fields: partialTC +func (_m *EventHandler) OnPartialTcCreated(partialTC *hotstuff.PartialTcCreated) error { + ret := _m.Called(partialTC) var r0 error - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) error); ok { - r0 = rf(qc) + if rf, ok := ret.Get(0).(func(*hotstuff.PartialTcCreated) error); ok { + r0 = rf(partialTC) } else { r0 = ret.Error(0) } @@ -59,13 +62,41 @@ func (_m *EventHandler) OnReceiveProposal(proposal *model.Proposal) error { return r0 } -// Start provides a mock function with given fields: -func (_m *EventHandler) Start() error { - ret := _m.Called() +// OnReceiveQc provides a mock function with given fields: qc +func (_m *EventHandler) OnReceiveQc(qc *flow.QuorumCertificate) error { + ret := _m.Called(qc) var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) error); ok { + r0 = rf(qc) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// OnReceiveTc provides a mock function with given fields: tc +func (_m *EventHandler) OnReceiveTc(tc *flow.TimeoutCertificate) error { + ret := _m.Called(tc) + + var r0 error + if rf, ok := ret.Get(0).(func(*flow.TimeoutCertificate) error); ok { + r0 = rf(tc) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Start provides a mock function with given fields: ctx +func (_m *EventHandler) Start(ctx context.Context) error { + ret := _m.Called(ctx) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) } else { r0 = ret.Error(0) } diff --git a/consensus/hotstuff/mocks/event_handler_v2.go b/consensus/hotstuff/mocks/event_handler_v2.go deleted file mode 100644 index 552e454daca..00000000000 --- a/consensus/hotstuff/mocks/event_handler_v2.go +++ /dev/null @@ -1,90 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" - - model "github.com/onflow/flow-go/consensus/hotstuff/model" - - time "time" -) - -// EventHandlerV2 is an autogenerated mock type for the EventHandlerV2 type -type EventHandlerV2 struct { - mock.Mock -} - -// OnLocalTimeout provides a mock function with given fields: -func (_m *EventHandlerV2) OnLocalTimeout() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// OnQCConstructed provides a mock function with given fields: qc -func (_m *EventHandlerV2) OnQCConstructed(qc *flow.QuorumCertificate) error { - ret := _m.Called(qc) - - var r0 error - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) error); ok { - r0 = rf(qc) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// OnReceiveProposal provides a mock function with given fields: proposal -func (_m *EventHandlerV2) OnReceiveProposal(proposal *model.Proposal) error { - ret := _m.Called(proposal) - - var r0 error - if rf, ok := ret.Get(0).(func(*model.Proposal) error); ok { - r0 = rf(proposal) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Start provides a mock function with given fields: -func (_m *EventHandlerV2) Start() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// TimeoutChannel provides a mock function with given fields: -func (_m *EventHandlerV2) TimeoutChannel() <-chan time.Time { - ret := _m.Called() - - var r0 <-chan time.Time - if rf, ok := ret.Get(0).(func() <-chan time.Time); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan time.Time) - } - } - - return r0 -} diff --git a/consensus/hotstuff/mocks/event_loop.go b/consensus/hotstuff/mocks/event_loop.go index c561625f3e0..91c804e491c 100644 --- a/consensus/hotstuff/mocks/event_loop.go +++ b/consensus/hotstuff/mocks/event_loop.go @@ -8,6 +8,8 @@ import ( irrecoverable "github.com/onflow/flow-go/module/irrecoverable" mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // EventLoop is an autogenerated mock type for the EventLoop type @@ -31,6 +33,31 @@ func (_m *EventLoop) Done() <-chan struct{} { return r0 } +// OnNewQcDiscovered provides a mock function with given fields: certificate +func (_m *EventLoop) OnNewQcDiscovered(certificate *flow.QuorumCertificate) { + _m.Called(certificate) +} + +// OnNewTcDiscovered provides a mock function with given fields: certificate +func (_m *EventLoop) OnNewTcDiscovered(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnPartialTcCreated provides a mock function with given fields: view, newestQC, lastViewTC +func (_m *EventLoop) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + _m.Called(view, newestQC, lastViewTC) +} + +// OnQcConstructedFromVotes provides a mock function with given fields: _a0 +func (_m *EventLoop) OnQcConstructedFromVotes(_a0 *flow.QuorumCertificate) { + _m.Called(_a0) +} + +// OnTcConstructedFromTimeouts provides a mock function with given fields: certificate +func (_m *EventLoop) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + // Ready provides a mock function with given fields: func (_m *EventLoop) Ready() <-chan struct{} { ret := _m.Called() @@ -52,25 +79,9 @@ func (_m *EventLoop) Start(_a0 irrecoverable.SignalerContext) { _m.Called(_a0) } -// SubmitProposal provides a mock function with given fields: proposal, parentView -func (_m *EventLoop) SubmitProposal(proposal *flow.Header, parentView uint64) <-chan struct{} { - ret := _m.Called(proposal, parentView) - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func(*flow.Header, uint64) <-chan struct{}); ok { - r0 = rf(proposal, parentView) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - -// SubmitTrustedQC provides a mock function with given fields: qc -func (_m *EventLoop) SubmitTrustedQC(qc *flow.QuorumCertificate) { - _m.Called(qc) +// SubmitProposal provides a mock function with given fields: proposal +func (_m *EventLoop) SubmitProposal(proposal *model.Proposal) { + _m.Called(proposal) } type mockConstructorTestingTNewEventLoop interface { diff --git a/consensus/hotstuff/mocks/forks.go b/consensus/hotstuff/mocks/forks.go index 6203a0b19ca..1c8b8ffb87b 100644 --- a/consensus/hotstuff/mocks/forks.go +++ b/consensus/hotstuff/mocks/forks.go @@ -15,27 +15,13 @@ type Forks struct { mock.Mock } -// AddBlock provides a mock function with given fields: block -func (_m *Forks) AddBlock(block *model.Block) error { - ret := _m.Called(block) +// AddProposal provides a mock function with given fields: proposal +func (_m *Forks) AddProposal(proposal *model.Proposal) error { + ret := _m.Called(proposal) var r0 error - if rf, ok := ret.Get(0).(func(*model.Block) error); ok { - r0 = rf(block) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// AddQC provides a mock function with given fields: qc -func (_m *Forks) AddQC(qc *flow.QuorumCertificate) error { - ret := _m.Called(qc) - - var r0 error - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) error); ok { - r0 = rf(qc) + if rf, ok := ret.Get(0).(func(*model.Proposal) error); ok { + r0 = rf(proposal) } else { r0 = ret.Error(0) } @@ -73,16 +59,16 @@ func (_m *Forks) FinalizedView() uint64 { return r0 } -// GetBlock provides a mock function with given fields: id -func (_m *Forks) GetBlock(id flow.Identifier) (*model.Block, bool) { +// GetProposal provides a mock function with given fields: id +func (_m *Forks) GetProposal(id flow.Identifier) (*model.Proposal, bool) { ret := _m.Called(id) - var r0 *model.Block - if rf, ok := ret.Get(0).(func(flow.Identifier) *model.Block); ok { + var r0 *model.Proposal + if rf, ok := ret.Get(0).(func(flow.Identifier) *model.Proposal); ok { r0 = rf(id) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Block) + r0 = ret.Get(0).(*model.Proposal) } } @@ -96,68 +82,36 @@ func (_m *Forks) GetBlock(id flow.Identifier) (*model.Block, bool) { return r0, r1 } -// GetBlocksForView provides a mock function with given fields: view -func (_m *Forks) GetBlocksForView(view uint64) []*model.Block { +// GetProposalsForView provides a mock function with given fields: view +func (_m *Forks) GetProposalsForView(view uint64) []*model.Proposal { ret := _m.Called(view) - var r0 []*model.Block - if rf, ok := ret.Get(0).(func(uint64) []*model.Block); ok { + var r0 []*model.Proposal + if rf, ok := ret.Get(0).(func(uint64) []*model.Proposal); ok { r0 = rf(view) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Block) + r0 = ret.Get(0).([]*model.Proposal) } } return r0 } -// IsSafeBlock provides a mock function with given fields: block -func (_m *Forks) IsSafeBlock(block *model.Block) bool { - ret := _m.Called(block) +// NewestView provides a mock function with given fields: +func (_m *Forks) NewestView() uint64 { + ret := _m.Called() - var r0 bool - if rf, ok := ret.Get(0).(func(*model.Block) bool); ok { - r0 = rf(block) + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() } else { - r0 = ret.Get(0).(bool) + r0 = ret.Get(0).(uint64) } return r0 } -// MakeForkChoice provides a mock function with given fields: curView -func (_m *Forks) MakeForkChoice(curView uint64) (*flow.QuorumCertificate, *model.Block, error) { - ret := _m.Called(curView) - - var r0 *flow.QuorumCertificate - if rf, ok := ret.Get(0).(func(uint64) *flow.QuorumCertificate); ok { - r0 = rf(curView) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.QuorumCertificate) - } - } - - var r1 *model.Block - if rf, ok := ret.Get(1).(func(uint64) *model.Block); ok { - r1 = rf(curView) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(*model.Block) - } - } - - var r2 error - if rf, ok := ret.Get(2).(func(uint64) error); ok { - r2 = rf(curView) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - type mockConstructorTestingTNewForks interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/pace_maker.go b/consensus/hotstuff/mocks/pace_maker.go index cfc2f91eb26..4f518dbb832 100644 --- a/consensus/hotstuff/mocks/pace_maker.go +++ b/consensus/hotstuff/mocks/pace_maker.go @@ -3,6 +3,8 @@ package mocks import ( + context "context" + flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" @@ -45,89 +47,105 @@ func (_m *PaceMaker) CurView() uint64 { return r0 } -// OnTimeout provides a mock function with given fields: -func (_m *PaceMaker) OnTimeout() *model.NewViewEvent { +// LastViewTC provides a mock function with given fields: +func (_m *PaceMaker) LastViewTC() *flow.TimeoutCertificate { ret := _m.Called() - var r0 *model.NewViewEvent - if rf, ok := ret.Get(0).(func() *model.NewViewEvent); ok { + var r0 *flow.TimeoutCertificate + if rf, ok := ret.Get(0).(func() *flow.TimeoutCertificate); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.NewViewEvent) + r0 = ret.Get(0).(*flow.TimeoutCertificate) } } return r0 } -// Start provides a mock function with given fields: -func (_m *PaceMaker) Start() { - _m.Called() -} - -// TimeoutChannel provides a mock function with given fields: -func (_m *PaceMaker) TimeoutChannel() <-chan time.Time { +// NewestQC provides a mock function with given fields: +func (_m *PaceMaker) NewestQC() *flow.QuorumCertificate { ret := _m.Called() - var r0 <-chan time.Time - if rf, ok := ret.Get(0).(func() <-chan time.Time); ok { + var r0 *flow.QuorumCertificate + if rf, ok := ret.Get(0).(func() *flow.QuorumCertificate); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan time.Time) + r0 = ret.Get(0).(*flow.QuorumCertificate) } } return r0 } -// UpdateCurViewWithBlock provides a mock function with given fields: block, isLeaderForNextView -func (_m *PaceMaker) UpdateCurViewWithBlock(block *model.Block, isLeaderForNextView bool) (*model.NewViewEvent, bool) { - ret := _m.Called(block, isLeaderForNextView) +// ProcessQC provides a mock function with given fields: qc +func (_m *PaceMaker) ProcessQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, error) { + ret := _m.Called(qc) var r0 *model.NewViewEvent - if rf, ok := ret.Get(0).(func(*model.Block, bool) *model.NewViewEvent); ok { - r0 = rf(block, isLeaderForNextView) + if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) *model.NewViewEvent); ok { + r0 = rf(qc) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*model.NewViewEvent) } } - var r1 bool - if rf, ok := ret.Get(1).(func(*model.Block, bool) bool); ok { - r1 = rf(block, isLeaderForNextView) + var r1 error + if rf, ok := ret.Get(1).(func(*flow.QuorumCertificate) error); ok { + r1 = rf(qc) } else { - r1 = ret.Get(1).(bool) + r1 = ret.Error(1) } return r0, r1 } -// UpdateCurViewWithQC provides a mock function with given fields: qc -func (_m *PaceMaker) UpdateCurViewWithQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, bool) { - ret := _m.Called(qc) +// ProcessTC provides a mock function with given fields: tc +func (_m *PaceMaker) ProcessTC(tc *flow.TimeoutCertificate) (*model.NewViewEvent, error) { + ret := _m.Called(tc) var r0 *model.NewViewEvent - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) *model.NewViewEvent); ok { - r0 = rf(qc) + if rf, ok := ret.Get(0).(func(*flow.TimeoutCertificate) *model.NewViewEvent); ok { + r0 = rf(tc) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*model.NewViewEvent) } } - var r1 bool - if rf, ok := ret.Get(1).(func(*flow.QuorumCertificate) bool); ok { - r1 = rf(qc) + var r1 error + if rf, ok := ret.Get(1).(func(*flow.TimeoutCertificate) error); ok { + r1 = rf(tc) } else { - r1 = ret.Get(1).(bool) + r1 = ret.Error(1) } return r0, r1 } +// Start provides a mock function with given fields: ctx +func (_m *PaceMaker) Start(ctx context.Context) { + _m.Called(ctx) +} + +// TimeoutChannel provides a mock function with given fields: +func (_m *PaceMaker) TimeoutChannel() <-chan time.Time { + ret := _m.Called() + + var r0 <-chan time.Time + if rf, ok := ret.Get(0).(func() <-chan time.Time); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan time.Time) + } + } + + return r0 +} + type mockConstructorTestingTNewPaceMaker interface { mock.TestingT Cleanup(func()) diff --git a/consensus/hotstuff/mocks/packer.go b/consensus/hotstuff/mocks/packer.go index 1afc7d54d11..462fbd7704a 100644 --- a/consensus/hotstuff/mocks/packer.go +++ b/consensus/hotstuff/mocks/packer.go @@ -14,13 +14,13 @@ type Packer struct { mock.Mock } -// Pack provides a mock function with given fields: blockID, sig -func (_m *Packer) Pack(blockID flow.Identifier, sig *hotstuff.BlockSignatureData) ([]byte, []byte, error) { - ret := _m.Called(blockID, sig) +// Pack provides a mock function with given fields: view, sig +func (_m *Packer) Pack(view uint64, sig *hotstuff.BlockSignatureData) ([]byte, []byte, error) { + ret := _m.Called(view, sig) var r0 []byte - if rf, ok := ret.Get(0).(func(flow.Identifier, *hotstuff.BlockSignatureData) []byte); ok { - r0 = rf(blockID, sig) + if rf, ok := ret.Get(0).(func(uint64, *hotstuff.BlockSignatureData) []byte); ok { + r0 = rf(view, sig) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) @@ -28,8 +28,8 @@ func (_m *Packer) Pack(blockID flow.Identifier, sig *hotstuff.BlockSignatureData } var r1 []byte - if rf, ok := ret.Get(1).(func(flow.Identifier, *hotstuff.BlockSignatureData) []byte); ok { - r1 = rf(blockID, sig) + if rf, ok := ret.Get(1).(func(uint64, *hotstuff.BlockSignatureData) []byte); ok { + r1 = rf(view, sig) } else { if ret.Get(1) != nil { r1 = ret.Get(1).([]byte) @@ -37,8 +37,8 @@ func (_m *Packer) Pack(blockID flow.Identifier, sig *hotstuff.BlockSignatureData } var r2 error - if rf, ok := ret.Get(2).(func(flow.Identifier, *hotstuff.BlockSignatureData) error); ok { - r2 = rf(blockID, sig) + if rf, ok := ret.Get(2).(func(uint64, *hotstuff.BlockSignatureData) error); ok { + r2 = rf(view, sig) } else { r2 = ret.Error(2) } diff --git a/consensus/hotstuff/mocks/persister.go b/consensus/hotstuff/mocks/persister.go index 36e14616fd2..5743c8d9fd3 100644 --- a/consensus/hotstuff/mocks/persister.go +++ b/consensus/hotstuff/mocks/persister.go @@ -2,22 +2,27 @@ package mocks -import mock "github.com/stretchr/testify/mock" +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + mock "github.com/stretchr/testify/mock" +) // Persister is an autogenerated mock type for the Persister type type Persister struct { mock.Mock } -// GetStarted provides a mock function with given fields: -func (_m *Persister) GetStarted() (uint64, error) { +// GetLivenessData provides a mock function with given fields: +func (_m *Persister) GetLivenessData() (*hotstuff.LivenessData, error) { ret := _m.Called() - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { + var r0 *hotstuff.LivenessData + if rf, ok := ret.Get(0).(func() *hotstuff.LivenessData); ok { r0 = rf() } else { - r0 = ret.Get(0).(uint64) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*hotstuff.LivenessData) + } } var r1 error @@ -30,15 +35,17 @@ func (_m *Persister) GetStarted() (uint64, error) { return r0, r1 } -// GetVoted provides a mock function with given fields: -func (_m *Persister) GetVoted() (uint64, error) { +// GetSafetyData provides a mock function with given fields: +func (_m *Persister) GetSafetyData() (*hotstuff.SafetyData, error) { ret := _m.Called() - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { + var r0 *hotstuff.SafetyData + if rf, ok := ret.Get(0).(func() *hotstuff.SafetyData); ok { r0 = rf() } else { - r0 = ret.Get(0).(uint64) + if ret.Get(0) != nil { + r0 = ret.Get(0).(*hotstuff.SafetyData) + } } var r1 error @@ -51,13 +58,13 @@ func (_m *Persister) GetVoted() (uint64, error) { return r0, r1 } -// PutStarted provides a mock function with given fields: view -func (_m *Persister) PutStarted(view uint64) error { - ret := _m.Called(view) +// PutLivenessData provides a mock function with given fields: livenessData +func (_m *Persister) PutLivenessData(livenessData *hotstuff.LivenessData) error { + ret := _m.Called(livenessData) var r0 error - if rf, ok := ret.Get(0).(func(uint64) error); ok { - r0 = rf(view) + if rf, ok := ret.Get(0).(func(*hotstuff.LivenessData) error); ok { + r0 = rf(livenessData) } else { r0 = ret.Error(0) } @@ -65,13 +72,13 @@ func (_m *Persister) PutStarted(view uint64) error { return r0 } -// PutVoted provides a mock function with given fields: view -func (_m *Persister) PutVoted(view uint64) error { - ret := _m.Called(view) +// PutSafetyData provides a mock function with given fields: safetyData +func (_m *Persister) PutSafetyData(safetyData *hotstuff.SafetyData) error { + ret := _m.Called(safetyData) var r0 error - if rf, ok := ret.Get(0).(func(uint64) error); ok { - r0 = rf(view) + if rf, ok := ret.Get(0).(func(*hotstuff.SafetyData) error); ok { + r0 = rf(safetyData) } else { r0 = ret.Error(0) } diff --git a/consensus/hotstuff/mocks/random_beacon_signer.go b/consensus/hotstuff/mocks/random_beacon_signer.go deleted file mode 100644 index e5907d74ade..00000000000 --- a/consensus/hotstuff/mocks/random_beacon_signer.go +++ /dev/null @@ -1,109 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - crypto "github.com/onflow/flow-go/crypto" - - mock "github.com/stretchr/testify/mock" -) - -// RandomBeaconSigner is an autogenerated mock type for the RandomBeaconSigner type -type RandomBeaconSigner struct { - mock.Mock -} - -// EnoughShares provides a mock function with given fields: -func (_m *RandomBeaconSigner) EnoughShares() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Reconstruct provides a mock function with given fields: -func (_m *RandomBeaconSigner) Reconstruct() (crypto.Signature, error) { - ret := _m.Called() - - var r0 crypto.Signature - if rf, ok := ret.Get(0).(func() crypto.Signature); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(crypto.Signature) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// SignShare provides a mock function with given fields: -func (_m *RandomBeaconSigner) SignShare() (crypto.Signature, error) { - ret := _m.Called() - - var r0 crypto.Signature - if rf, ok := ret.Get(0).(func() crypto.Signature); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(crypto.Signature) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// TrustedAdd provides a mock function with given fields: signerIndex, share -func (_m *RandomBeaconSigner) TrustedAdd(signerIndex int, share crypto.Signature) (bool, error) { - ret := _m.Called(signerIndex, share) - - var r0 bool - if rf, ok := ret.Get(0).(func(int, crypto.Signature) bool); ok { - r0 = rf(signerIndex, share) - } else { - r0 = ret.Get(0).(bool) - } - - var r1 error - if rf, ok := ret.Get(1).(func(int, crypto.Signature) error); ok { - r1 = rf(signerIndex, share) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Verify provides a mock function with given fields: signerIndex, share -func (_m *RandomBeaconSigner) Verify(signerIndex int, share crypto.Signature) error { - ret := _m.Called(signerIndex, share) - - var r0 error - if rf, ok := ret.Get(0).(func(int, crypto.Signature) error); ok { - r0 = rf(signerIndex, share) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/consensus/hotstuff/mocks/replicas.go b/consensus/hotstuff/mocks/replicas.go new file mode 100644 index 00000000000..b140014e7b0 --- /dev/null +++ b/consensus/hotstuff/mocks/replicas.go @@ -0,0 +1,180 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" +) + +// Replicas is an autogenerated mock type for the Replicas type +type Replicas struct { + mock.Mock +} + +// DKG provides a mock function with given fields: view +func (_m *Replicas) DKG(view uint64) (hotstuff.DKG, error) { + ret := _m.Called(view) + + var r0 hotstuff.DKG + if rf, ok := ret.Get(0).(func(uint64) hotstuff.DKG); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(hotstuff.DKG) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentitiesByEpoch provides a mock function with given fields: view +func (_m *Replicas) IdentitiesByEpoch(view uint64) (flow.IdentityList, error) { + ret := _m.Called(view) + + var r0 flow.IdentityList + if rf, ok := ret.Get(0).(func(uint64) flow.IdentityList); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.IdentityList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// IdentityByEpoch provides a mock function with given fields: view, participantID +func (_m *Replicas) IdentityByEpoch(view uint64, participantID flow.Identifier) (*flow.Identity, error) { + ret := _m.Called(view, participantID) + + var r0 *flow.Identity + if rf, ok := ret.Get(0).(func(uint64, flow.Identifier) *flow.Identity); ok { + r0 = rf(view, participantID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Identity) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64, flow.Identifier) error); ok { + r1 = rf(view, participantID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LeaderForView provides a mock function with given fields: view +func (_m *Replicas) LeaderForView(view uint64) (flow.Identifier, error) { + ret := _m.Called(view) + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func(uint64) flow.Identifier); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// QuorumThresholdForView provides a mock function with given fields: view +func (_m *Replicas) QuorumThresholdForView(view uint64) (uint64, error) { + ret := _m.Called(view) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(view) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Self provides a mock function with given fields: +func (_m *Replicas) Self() flow.Identifier { + ret := _m.Called() + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func() flow.Identifier); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + return r0 +} + +// TimeoutThresholdForView provides a mock function with given fields: view +func (_m *Replicas) TimeoutThresholdForView(view uint64) (uint64, error) { + ret := _m.Called(view) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(uint64) uint64); ok { + r0 = rf(view) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewReplicas interface { + mock.TestingT + Cleanup(func()) +} + +// NewReplicas creates a new instance of Replicas. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewReplicas(t mockConstructorTestingTNewReplicas) *Replicas { + mock := &Replicas{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/safety_rules.go b/consensus/hotstuff/mocks/safety_rules.go new file mode 100644 index 00000000000..c8c06e465bc --- /dev/null +++ b/consensus/hotstuff/mocks/safety_rules.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// SafetyRules is an autogenerated mock type for the SafetyRules type +type SafetyRules struct { + mock.Mock +} + +// ProduceTimeout provides a mock function with given fields: curView, newestQC, lastViewTC +func (_m *SafetyRules) ProduceTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + ret := _m.Called(curView, newestQC, lastViewTC) + + var r0 *model.TimeoutObject + if rf, ok := ret.Get(0).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) *model.TimeoutObject); ok { + r0 = rf(curView, newestQC, lastViewTC) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.TimeoutObject) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) error); ok { + r1 = rf(curView, newestQC, lastViewTC) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProduceVote provides a mock function with given fields: proposal, curView +func (_m *SafetyRules) ProduceVote(proposal *model.Proposal, curView uint64) (*model.Vote, error) { + ret := _m.Called(proposal, curView) + + var r0 *model.Vote + if rf, ok := ret.Get(0).(func(*model.Proposal, uint64) *model.Vote); ok { + r0 = rf(proposal, curView) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Vote) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.Proposal, uint64) error); ok { + r1 = rf(proposal, curView) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewSafetyRules interface { + mock.TestingT + Cleanup(func()) +} + +// NewSafetyRules creates a new instance of SafetyRules. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewSafetyRules(t mockConstructorTestingTNewSafetyRules) *SafetyRules { + mock := &SafetyRules{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/signer.go b/consensus/hotstuff/mocks/signer.go index 7cb2df30f30..1a36ebab53e 100644 --- a/consensus/hotstuff/mocks/signer.go +++ b/consensus/hotstuff/mocks/signer.go @@ -3,8 +3,11 @@ package mocks import ( - model "github.com/onflow/flow-go/consensus/hotstuff/model" + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // Signer is an autogenerated mock type for the Signer type @@ -35,6 +38,29 @@ func (_m *Signer) CreateProposal(block *model.Block) (*model.Proposal, error) { return r0, r1 } +// CreateTimeout provides a mock function with given fields: curView, newestQC, lastViewTC +func (_m *Signer) CreateTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + ret := _m.Called(curView, newestQC, lastViewTC) + + var r0 *model.TimeoutObject + if rf, ok := ret.Get(0).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) *model.TimeoutObject); ok { + r0 = rf(curView, newestQC, lastViewTC) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.TimeoutObject) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) error); ok { + r1 = rf(curView, newestQC, lastViewTC) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // CreateVote provides a mock function with given fields: block func (_m *Signer) CreateVote(block *model.Block) (*model.Vote, error) { ret := _m.Called(block) diff --git a/consensus/hotstuff/mocks/timeout_aggregator.go b/consensus/hotstuff/mocks/timeout_aggregator.go new file mode 100644 index 00000000000..984b66932a3 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_aggregator.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// TimeoutAggregator is an autogenerated mock type for the TimeoutAggregator type +type TimeoutAggregator struct { + mock.Mock +} + +// AddTimeout provides a mock function with given fields: timeoutObject +func (_m *TimeoutAggregator) AddTimeout(timeoutObject *model.TimeoutObject) { + _m.Called(timeoutObject) +} + +// Done provides a mock function with given fields: +func (_m *TimeoutAggregator) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// PruneUpToView provides a mock function with given fields: lowestRetainedView +func (_m *TimeoutAggregator) PruneUpToView(lowestRetainedView uint64) { + _m.Called(lowestRetainedView) +} + +// Ready provides a mock function with given fields: +func (_m *TimeoutAggregator) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *TimeoutAggregator) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewTimeoutAggregator interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutAggregator creates a new instance of TimeoutAggregator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutAggregator(t mockConstructorTestingTNewTimeoutAggregator) *TimeoutAggregator { + mock := &TimeoutAggregator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collector.go b/consensus/hotstuff/mocks/timeout_collector.go new file mode 100644 index 00000000000..827cff717f3 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_collector.go @@ -0,0 +1,56 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutCollector is an autogenerated mock type for the TimeoutCollector type +type TimeoutCollector struct { + mock.Mock +} + +// AddTimeout provides a mock function with given fields: timeoutObject +func (_m *TimeoutCollector) AddTimeout(timeoutObject *model.TimeoutObject) error { + ret := _m.Called(timeoutObject) + + var r0 error + if rf, ok := ret.Get(0).(func(*model.TimeoutObject) error); ok { + r0 = rf(timeoutObject) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// View provides a mock function with given fields: +func (_m *TimeoutCollector) View() uint64 { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + +type mockConstructorTestingTNewTimeoutCollector interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutCollector creates a new instance of TimeoutCollector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutCollector(t mockConstructorTestingTNewTimeoutCollector) *TimeoutCollector { + mock := &TimeoutCollector{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collector_consumer.go b/consensus/hotstuff/mocks/timeout_collector_consumer.go new file mode 100644 index 00000000000..33a45aacae6 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_collector_consumer.go @@ -0,0 +1,49 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" +) + +// TimeoutCollectorConsumer is an autogenerated mock type for the TimeoutCollectorConsumer type +type TimeoutCollectorConsumer struct { + mock.Mock +} + +// OnNewQcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutCollectorConsumer) OnNewQcDiscovered(certificate *flow.QuorumCertificate) { + _m.Called(certificate) +} + +// OnNewTcDiscovered provides a mock function with given fields: certificate +func (_m *TimeoutCollectorConsumer) OnNewTcDiscovered(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +// OnPartialTcCreated provides a mock function with given fields: view, newestQC, lastViewTC +func (_m *TimeoutCollectorConsumer) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + _m.Called(view, newestQC, lastViewTC) +} + +// OnTcConstructedFromTimeouts provides a mock function with given fields: certificate +func (_m *TimeoutCollectorConsumer) OnTcConstructedFromTimeouts(certificate *flow.TimeoutCertificate) { + _m.Called(certificate) +} + +type mockConstructorTestingTNewTimeoutCollectorConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutCollectorConsumer creates a new instance of TimeoutCollectorConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutCollectorConsumer(t mockConstructorTestingTNewTimeoutCollectorConsumer) *TimeoutCollectorConsumer { + mock := &TimeoutCollectorConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collector_factory.go b/consensus/hotstuff/mocks/timeout_collector_factory.go new file mode 100644 index 00000000000..97c8e8fae03 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_collector_factory.go @@ -0,0 +1,51 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutCollectorFactory is an autogenerated mock type for the TimeoutCollectorFactory type +type TimeoutCollectorFactory struct { + mock.Mock +} + +// Create provides a mock function with given fields: view +func (_m *TimeoutCollectorFactory) Create(view uint64) (hotstuff.TimeoutCollector, error) { + ret := _m.Called(view) + + var r0 hotstuff.TimeoutCollector + if rf, ok := ret.Get(0).(func(uint64) hotstuff.TimeoutCollector); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(hotstuff.TimeoutCollector) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewTimeoutCollectorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutCollectorFactory creates a new instance of TimeoutCollectorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutCollectorFactory(t mockConstructorTestingTNewTimeoutCollectorFactory) *TimeoutCollectorFactory { + mock := &TimeoutCollectorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_collectors.go b/consensus/hotstuff/mocks/timeout_collectors.go new file mode 100644 index 00000000000..cf1b986affb --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_collectors.go @@ -0,0 +1,63 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutCollectors is an autogenerated mock type for the TimeoutCollectors type +type TimeoutCollectors struct { + mock.Mock +} + +// GetOrCreateCollector provides a mock function with given fields: view +func (_m *TimeoutCollectors) GetOrCreateCollector(view uint64) (hotstuff.TimeoutCollector, bool, error) { + ret := _m.Called(view) + + var r0 hotstuff.TimeoutCollector + if rf, ok := ret.Get(0).(func(uint64) hotstuff.TimeoutCollector); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(hotstuff.TimeoutCollector) + } + } + + var r1 bool + if rf, ok := ret.Get(1).(func(uint64) bool); ok { + r1 = rf(view) + } else { + r1 = ret.Get(1).(bool) + } + + var r2 error + if rf, ok := ret.Get(2).(func(uint64) error); ok { + r2 = rf(view) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// PruneUpToView provides a mock function with given fields: lowestRetainedView +func (_m *TimeoutCollectors) PruneUpToView(lowestRetainedView uint64) { + _m.Called(lowestRetainedView) +} + +type mockConstructorTestingTNewTimeoutCollectors interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutCollectors creates a new instance of TimeoutCollectors. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutCollectors(t mockConstructorTestingTNewTimeoutCollectors) *TimeoutCollectors { + mock := &TimeoutCollectors{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_processor.go b/consensus/hotstuff/mocks/timeout_processor.go new file mode 100644 index 00000000000..f7fd2c610be --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_processor.go @@ -0,0 +1,42 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + model "github.com/onflow/flow-go/consensus/hotstuff/model" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutProcessor is an autogenerated mock type for the TimeoutProcessor type +type TimeoutProcessor struct { + mock.Mock +} + +// Process provides a mock function with given fields: timeout +func (_m *TimeoutProcessor) Process(timeout *model.TimeoutObject) error { + ret := _m.Called(timeout) + + var r0 error + if rf, ok := ret.Get(0).(func(*model.TimeoutObject) error); ok { + r0 = rf(timeout) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewTimeoutProcessor interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutProcessor creates a new instance of TimeoutProcessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutProcessor(t mockConstructorTestingTNewTimeoutProcessor) *TimeoutProcessor { + mock := &TimeoutProcessor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_processor_factory.go b/consensus/hotstuff/mocks/timeout_processor_factory.go new file mode 100644 index 00000000000..a3d98076d91 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_processor_factory.go @@ -0,0 +1,51 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + mock "github.com/stretchr/testify/mock" +) + +// TimeoutProcessorFactory is an autogenerated mock type for the TimeoutProcessorFactory type +type TimeoutProcessorFactory struct { + mock.Mock +} + +// Create provides a mock function with given fields: view +func (_m *TimeoutProcessorFactory) Create(view uint64) (hotstuff.TimeoutProcessor, error) { + ret := _m.Called(view) + + var r0 hotstuff.TimeoutProcessor + if rf, ok := ret.Get(0).(func(uint64) hotstuff.TimeoutProcessor); ok { + r0 = rf(view) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(hotstuff.TimeoutProcessor) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(view) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewTimeoutProcessorFactory interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutProcessorFactory creates a new instance of TimeoutProcessorFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutProcessorFactory(t mockConstructorTestingTNewTimeoutProcessorFactory) *TimeoutProcessorFactory { + mock := &TimeoutProcessorFactory{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/timeout_signature_aggregator.go b/consensus/hotstuff/mocks/timeout_signature_aggregator.go new file mode 100644 index 00000000000..35a25149c95 --- /dev/null +++ b/consensus/hotstuff/mocks/timeout_signature_aggregator.go @@ -0,0 +1,113 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mocks + +import ( + crypto "github.com/onflow/flow-go/crypto" + flow "github.com/onflow/flow-go/model/flow" + + hotstuff "github.com/onflow/flow-go/consensus/hotstuff" + + mock "github.com/stretchr/testify/mock" +) + +// TimeoutSignatureAggregator is an autogenerated mock type for the TimeoutSignatureAggregator type +type TimeoutSignatureAggregator struct { + mock.Mock +} + +// Aggregate provides a mock function with given fields: +func (_m *TimeoutSignatureAggregator) Aggregate() ([]hotstuff.TimeoutSignerInfo, crypto.Signature, error) { + ret := _m.Called() + + var r0 []hotstuff.TimeoutSignerInfo + if rf, ok := ret.Get(0).(func() []hotstuff.TimeoutSignerInfo); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]hotstuff.TimeoutSignerInfo) + } + } + + var r1 crypto.Signature + if rf, ok := ret.Get(1).(func() crypto.Signature); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(crypto.Signature) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func() error); ok { + r2 = rf() + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// TotalWeight provides a mock function with given fields: +func (_m *TimeoutSignatureAggregator) TotalWeight() uint64 { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + +// VerifyAndAdd provides a mock function with given fields: signerID, sig, newestQCView +func (_m *TimeoutSignatureAggregator) VerifyAndAdd(signerID flow.Identifier, sig crypto.Signature, newestQCView uint64) (uint64, error) { + ret := _m.Called(signerID, sig, newestQCView) + + var r0 uint64 + if rf, ok := ret.Get(0).(func(flow.Identifier, crypto.Signature, uint64) uint64); ok { + r0 = rf(signerID, sig, newestQCView) + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(flow.Identifier, crypto.Signature, uint64) error); ok { + r1 = rf(signerID, sig, newestQCView) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// View provides a mock function with given fields: +func (_m *TimeoutSignatureAggregator) View() uint64 { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + +type mockConstructorTestingTNewTimeoutSignatureAggregator interface { + mock.TestingT + Cleanup(func()) +} + +// NewTimeoutSignatureAggregator creates a new instance of TimeoutSignatureAggregator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewTimeoutSignatureAggregator(t mockConstructorTestingTNewTimeoutSignatureAggregator) *TimeoutSignatureAggregator { + mock := &TimeoutSignatureAggregator{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/consensus/hotstuff/mocks/validator.go b/consensus/hotstuff/mocks/validator.go index a23c5a63732..ae604b23f41 100644 --- a/consensus/hotstuff/mocks/validator.go +++ b/consensus/hotstuff/mocks/validator.go @@ -29,13 +29,13 @@ func (_m *Validator) ValidateProposal(proposal *model.Proposal) error { return r0 } -// ValidateQC provides a mock function with given fields: qc, block -func (_m *Validator) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) error { - ret := _m.Called(qc, block) +// ValidateQC provides a mock function with given fields: qc +func (_m *Validator) ValidateQC(qc *flow.QuorumCertificate) error { + ret := _m.Called(qc) var r0 error - if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate, *model.Block) error); ok { - r0 = rf(qc, block) + if rf, ok := ret.Get(0).(func(*flow.QuorumCertificate) error); ok { + r0 = rf(qc) } else { r0 = ret.Error(0) } @@ -43,13 +43,27 @@ func (_m *Validator) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) return r0 } -// ValidateVote provides a mock function with given fields: vote, block -func (_m *Validator) ValidateVote(vote *model.Vote, block *model.Block) (*flow.Identity, error) { - ret := _m.Called(vote, block) +// ValidateTC provides a mock function with given fields: tc +func (_m *Validator) ValidateTC(tc *flow.TimeoutCertificate) error { + ret := _m.Called(tc) + + var r0 error + if rf, ok := ret.Get(0).(func(*flow.TimeoutCertificate) error); ok { + r0 = rf(tc) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ValidateVote provides a mock function with given fields: vote +func (_m *Validator) ValidateVote(vote *model.Vote) (*flow.Identity, error) { + ret := _m.Called(vote) var r0 *flow.Identity - if rf, ok := ret.Get(0).(func(*model.Vote, *model.Block) *flow.Identity); ok { - r0 = rf(vote, block) + if rf, ok := ret.Get(0).(func(*model.Vote) *flow.Identity); ok { + r0 = rf(vote) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*flow.Identity) @@ -57,8 +71,8 @@ func (_m *Validator) ValidateVote(vote *model.Vote, block *model.Block) (*flow.I } var r1 error - if rf, ok := ret.Get(1).(func(*model.Vote, *model.Block) error); ok { - r1 = rf(vote, block) + if rf, ok := ret.Get(1).(func(*model.Vote) error); ok { + r1 = rf(vote) } else { r1 = ret.Error(1) } diff --git a/consensus/hotstuff/mocks/verifier.go b/consensus/hotstuff/mocks/verifier.go index aa2aacb7614..92d02614abe 100644 --- a/consensus/hotstuff/mocks/verifier.go +++ b/consensus/hotstuff/mocks/verifier.go @@ -6,8 +6,6 @@ import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" - - model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // Verifier is an autogenerated mock type for the Verifier type @@ -15,13 +13,27 @@ type Verifier struct { mock.Mock } -// VerifyQC provides a mock function with given fields: signers, sigData, block -func (_m *Verifier) VerifyQC(signers flow.IdentityList, sigData []byte, block *model.Block) error { - ret := _m.Called(signers, sigData, block) +// VerifyQC provides a mock function with given fields: signers, sigData, view, blockID +func (_m *Verifier) VerifyQC(signers flow.IdentityList, sigData []byte, view uint64, blockID flow.Identifier) error { + ret := _m.Called(signers, sigData, view, blockID) + + var r0 error + if rf, ok := ret.Get(0).(func(flow.IdentityList, []byte, uint64, flow.Identifier) error); ok { + r0 = rf(signers, sigData, view, blockID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// VerifyTC provides a mock function with given fields: signers, sigData, view, highQCViews +func (_m *Verifier) VerifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64) error { + ret := _m.Called(signers, sigData, view, highQCViews) var r0 error - if rf, ok := ret.Get(0).(func(flow.IdentityList, []byte, *model.Block) error); ok { - r0 = rf(signers, sigData, block) + if rf, ok := ret.Get(0).(func(flow.IdentityList, []byte, uint64, []uint64) error); ok { + r0 = rf(signers, sigData, view, highQCViews) } else { r0 = ret.Error(0) } @@ -29,13 +41,13 @@ func (_m *Verifier) VerifyQC(signers flow.IdentityList, sigData []byte, block *m return r0 } -// VerifyVote provides a mock function with given fields: voter, sigData, block -func (_m *Verifier) VerifyVote(voter *flow.Identity, sigData []byte, block *model.Block) error { - ret := _m.Called(voter, sigData, block) +// VerifyVote provides a mock function with given fields: voter, sigData, view, blockID +func (_m *Verifier) VerifyVote(voter *flow.Identity, sigData []byte, view uint64, blockID flow.Identifier) error { + ret := _m.Called(voter, sigData, view, blockID) var r0 error - if rf, ok := ret.Get(0).(func(*flow.Identity, []byte, *model.Block) error); ok { - r0 = rf(voter, sigData, block) + if rf, ok := ret.Get(0).(func(*flow.Identity, []byte, uint64, flow.Identifier) error); ok { + r0 = rf(voter, sigData, view, blockID) } else { r0 = ret.Error(0) } diff --git a/consensus/hotstuff/mocks/vote_aggregator.go b/consensus/hotstuff/mocks/vote_aggregator.go index e511def15cf..1dac57219c9 100644 --- a/consensus/hotstuff/mocks/vote_aggregator.go +++ b/consensus/hotstuff/mocks/vote_aggregator.go @@ -15,17 +15,8 @@ type VoteAggregator struct { } // AddBlock provides a mock function with given fields: block -func (_m *VoteAggregator) AddBlock(block *model.Proposal) error { - ret := _m.Called(block) - - var r0 error - if rf, ok := ret.Get(0).(func(*model.Proposal) error); ok { - r0 = rf(block) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *VoteAggregator) AddBlock(block *model.Proposal) { + _m.Called(block) } // AddVote provides a mock function with given fields: vote diff --git a/consensus/hotstuff/mocks/vote_collector_state.go b/consensus/hotstuff/mocks/vote_collector_state.go deleted file mode 100644 index 6ea6c2cd305..00000000000 --- a/consensus/hotstuff/mocks/vote_collector_state.go +++ /dev/null @@ -1,57 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. - -package mocks - -import ( - hotstuff "github.com/onflow/flow-go/consensus/hotstuff" - mock "github.com/stretchr/testify/mock" - - model "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// VoteCollectorState is an autogenerated mock type for the VoteCollectorState type -type VoteCollectorState struct { - mock.Mock -} - -// AddVote provides a mock function with given fields: vote -func (_m *VoteCollectorState) AddVote(vote *model.Vote) error { - ret := _m.Called(vote) - - var r0 error - if rf, ok := ret.Get(0).(func(*model.Vote) error); ok { - r0 = rf(vote) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Status provides a mock function with given fields: -func (_m *VoteCollectorState) Status() hotstuff.VoteCollectorStatus { - ret := _m.Called() - - var r0 hotstuff.VoteCollectorStatus - if rf, ok := ret.Get(0).(func() hotstuff.VoteCollectorStatus); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(hotstuff.VoteCollectorStatus) - } - - return r0 -} - -// View provides a mock function with given fields: -func (_m *VoteCollectorState) View() uint64 { - ret := _m.Called() - - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint64) - } - - return r0 -} diff --git a/consensus/hotstuff/model/block.go b/consensus/hotstuff/model/block.go index a8f952ba11a..b8a3b155bc5 100644 --- a/consensus/hotstuff/model/block.go +++ b/consensus/hotstuff/model/block.go @@ -18,11 +18,10 @@ type Block struct { } // BlockFromFlow converts a flow header to a hotstuff block. -func BlockFromFlow(header *flow.Header, parentView uint64) *Block { - +func BlockFromFlow(header *flow.Header) *Block { qc := flow.QuorumCertificate{ BlockID: header.ParentID, - View: parentView, + View: header.ParentView, SignerIndices: header.ParentVoterIndices, SigData: header.ParentVoterSigData, } diff --git a/consensus/hotstuff/model/errors.go b/consensus/hotstuff/model/errors.go index 8dc0a92998b..cbc167384d5 100644 --- a/consensus/hotstuff/model/errors.go +++ b/consensus/hotstuff/model/errors.go @@ -10,14 +10,23 @@ import ( var ( ErrUnverifiableBlock = errors.New("block proposal can't be verified, because its view is above the finalized view, but its QC is below the finalized view") ErrInvalidSignature = errors.New("invalid signature") + // ErrViewForUnknownEpoch is returned when Epoch information is queried for a view that is + // outside of all cached epochs. This can happen when a query is made for a view in the + // next epoch, if that epoch is not committed yet. This can also happen when an + // old epoch is queried (>3 in the past), even if that epoch does exist in storage. + ErrViewForUnknownEpoch = fmt.Errorf("by-view query for unknown epoch") ) -// NoVoteError contains the reason of why the voter didn't vote for a block proposal. +// NoVoteError contains the reason why hotstuff.SafetyRules refused to generate a `Vote` for the current view. type NoVoteError struct { - Msg string + Err error } -func (e NoVoteError) Error() string { return e.Msg } +func (e NoVoteError) Error() string { return fmt.Sprintf("not voting - %s", e.Err.Error()) } + +func (e NoVoteError) Unwrap() error { + return e.Err +} // IsNoVoteError returns whether an error is NoVoteError func IsNoVoteError(err error) bool { @@ -25,6 +34,33 @@ func IsNoVoteError(err error) bool { return errors.As(err, &e) } +func NewNoVoteErrorf(msg string, args ...interface{}) error { + return NoVoteError{Err: fmt.Errorf(msg, args...)} +} + +// NoTimeoutError contains the reason why hotstuff.SafetyRules refused to generate a `TimeoutObject` [TO] for the current view. +type NoTimeoutError struct { + Err error +} + +func (e NoTimeoutError) Error() string { + return fmt.Sprintf("conditions not satisfied to generate valid TimeoutObject: %s", e.Err.Error()) +} + +func (e NoTimeoutError) Unwrap() error { + return e.Err +} + +// IsNoTimeoutError returns whether an error is NoTimeoutError +func IsNoTimeoutError(err error) bool { + var e NoTimeoutError + return errors.As(err, &e) +} + +func NewNoTimeoutErrorf(msg string, args ...interface{}) error { + return NoTimeoutError{Err: fmt.Errorf(msg, args...)} +} + // InvalidFormatError indicates that some data has an incompatible format. type InvalidFormatError struct { err error @@ -77,7 +113,7 @@ type MissingBlockError struct { } func (e MissingBlockError) Error() string { - return fmt.Sprintf("missing Block at view %d with ID %v", e.View, e.BlockID) + return fmt.Sprintf("missing Proposal at view %d with ID %v", e.View, e.BlockID) } // IsMissingBlockError returns whether an error is MissingBlockError @@ -86,6 +122,47 @@ func IsMissingBlockError(err error) bool { return errors.As(err, &e) } +// InvalidQCError indicates that the QC for block identified by `BlockID` and `View` is invalid +type InvalidQCError struct { + BlockID flow.Identifier + View uint64 + Err error +} + +func (e InvalidQCError) Error() string { + return fmt.Sprintf("invalid QC for block %x at view %d: %s", e.BlockID, e.View, e.Err.Error()) +} + +// IsInvalidQCError returns whether an error is InvalidQCError +func IsInvalidQCError(err error) bool { + var e InvalidQCError + return errors.As(err, &e) +} + +func (e InvalidQCError) Unwrap() error { + return e.Err +} + +// InvalidTCError indicates that the TC for view identified by `View` is invalid +type InvalidTCError struct { + View uint64 + Err error +} + +func (e InvalidTCError) Error() string { + return fmt.Sprintf("invalid TC at view %d: %s", e.View, e.Err.Error()) +} + +// IsInvalidTCError returns whether an error is InvalidQCError +func IsInvalidTCError(err error) bool { + var e InvalidTCError + return errors.As(err, &e) +} + +func (e InvalidTCError) Unwrap() error { + return e.Err +} + // InvalidBlockError indicates that the block with identifier `BlockID` is invalid type InvalidBlockError struct { BlockID flow.Identifier @@ -109,21 +186,19 @@ func (e InvalidBlockError) Unwrap() error { // InvalidVoteError indicates that the vote with identifier `VoteID` is invalid type InvalidVoteError struct { - VoteID flow.Identifier - View uint64 - Err error + Vote *Vote + Err error } func NewInvalidVoteErrorf(vote *Vote, msg string, args ...interface{}) error { return InvalidVoteError{ - VoteID: vote.ID(), - View: vote.View, - Err: fmt.Errorf(msg, args...), + Vote: vote, + Err: fmt.Errorf(msg, args...), } } func (e InvalidVoteError) Error() string { - return fmt.Sprintf("invalid vote %x for view %d: %s", e.VoteID, e.View, e.Err.Error()) + return fmt.Sprintf("invalid vote at view %d for block %x: %s", e.Vote.View, e.Vote.BlockID, e.Err.Error()) } // IsInvalidVoteError returns whether an error is InvalidVoteError @@ -132,6 +207,17 @@ func IsInvalidVoteError(err error) bool { return errors.As(err, &e) } +// AsInvalidVoteError determines whether the given error is a InvalidVoteError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidVoteError(err error) (*InvalidVoteError, bool) { + var e InvalidVoteError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + func (e InvalidVoteError) Unwrap() error { return e.Err } @@ -148,6 +234,11 @@ func (e ByzantineThresholdExceededError) Error() string { return e.Evidence } +func IsByzantineThresholdExceededError(err error) bool { + var target ByzantineThresholdExceededError + return errors.As(err, &target) +} + // DoubleVoteError indicates that a consensus replica has voted for two different // blocks, or has provided two semantically different votes for the same block. type DoubleVoteError struct { @@ -276,3 +367,82 @@ func IsInvalidSignerError(err error) bool { var e InvalidSignerError return errors.As(err, &e) } + +// DoubleTimeoutError indicates that a consensus replica has created two different +// timeout objects for same view. +type DoubleTimeoutError struct { + FirstTimeout *TimeoutObject + ConflictingTimeout *TimeoutObject + err error +} + +func (e DoubleTimeoutError) Error() string { + return e.err.Error() +} + +// IsDoubleTimeoutError returns whether an error is DoubleTimeoutError +func IsDoubleTimeoutError(err error) bool { + var e DoubleTimeoutError + return errors.As(err, &e) +} + +// AsDoubleTimeoutError determines whether the given error is a DoubleTimeoutError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsDoubleTimeoutError(err error) (*DoubleTimeoutError, bool) { + var e DoubleTimeoutError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + +func (e DoubleTimeoutError) Unwrap() error { + return e.err +} + +func NewDoubleTimeoutErrorf(firstTimeout, conflictingTimeout *TimeoutObject, msg string, args ...interface{}) error { + return DoubleTimeoutError{ + FirstTimeout: firstTimeout, + ConflictingTimeout: conflictingTimeout, + err: fmt.Errorf(msg, args...), + } +} + +// InvalidTimeoutError indicates that the embedded timeout object is invalid +type InvalidTimeoutError struct { + Timeout *TimeoutObject + Err error +} + +func NewInvalidTimeoutErrorf(timeout *TimeoutObject, msg string, args ...interface{}) error { + return InvalidTimeoutError{ + Timeout: timeout, + Err: fmt.Errorf(msg, args...), + } +} + +func (e InvalidTimeoutError) Error() string { + return fmt.Sprintf("invalid timeout %x for view %d: %s", e.Timeout.ID(), e.Timeout.View, e.Err.Error()) +} + +// IsInvalidTimeoutError returns whether an error is InvalidTimeoutError +func IsInvalidTimeoutError(err error) bool { + var e InvalidTimeoutError + return errors.As(err, &e) +} + +// AsInvalidTimeoutError determines whether the given error is a InvalidTimeoutError +// (potentially wrapped). It follows the same semantics as a checked type cast. +func AsInvalidTimeoutError(err error) (*InvalidTimeoutError, bool) { + var e InvalidTimeoutError + ok := errors.As(err, &e) + if ok { + return &e, true + } + return nil, false +} + +func (e InvalidTimeoutError) Unwrap() error { + return e.Err +} diff --git a/consensus/hotstuff/model/newview.go b/consensus/hotstuff/model/newview.go deleted file mode 100644 index fb104d4e990..00000000000 --- a/consensus/hotstuff/model/newview.go +++ /dev/null @@ -1,6 +0,0 @@ -package model - -// NewViewEvent is the new view event that contains the new view. -type NewViewEvent struct { - View uint64 -} diff --git a/consensus/hotstuff/model/proposal.go b/consensus/hotstuff/model/proposal.go index ba0a342df0b..538190906dd 100644 --- a/consensus/hotstuff/model/proposal.go +++ b/consensus/hotstuff/model/proposal.go @@ -7,8 +7,9 @@ import ( // Proposal represent a new proposed block within HotStuff (and thus a // a header in the bigger picture), signed by the proposer. type Proposal struct { - Block *Block - SigData []byte + Block *Block + SigData []byte + LastViewTC *flow.TimeoutCertificate } // ProposerVote extracts the proposer vote from the proposal @@ -23,13 +24,14 @@ func (p *Proposal) ProposerVote() *Vote { } // ProposalFromFlow turns a flow header into a hotstuff block type. -func ProposalFromFlow(header *flow.Header, parentView uint64) *Proposal { +func ProposalFromFlow(header *flow.Header) *Proposal { - block := BlockFromFlow(header, parentView) + block := BlockFromFlow(header) proposal := Proposal{ - Block: block, - SigData: header.ProposerSigData, + Block: block, + SigData: header.ProposerSigData, + LastViewTC: header.LastViewTC, } return &proposal @@ -44,10 +46,12 @@ func ProposalToFlow(proposal *Proposal) *flow.Header { PayloadHash: block.PayloadHash, Timestamp: block.Timestamp, View: block.View, + ParentView: block.QC.View, ParentVoterIndices: block.QC.SignerIndices, ParentVoterSigData: block.QC.SigData, ProposerID: block.ProposerID, ProposerSigData: proposal.SigData, + LastViewTC: proposal.LastViewTC, } return header diff --git a/consensus/hotstuff/model/timeout.go b/consensus/hotstuff/model/timeout.go index 37a6aeb3e55..51347e4b41f 100644 --- a/consensus/hotstuff/model/timeout.go +++ b/consensus/hotstuff/model/timeout.go @@ -1,29 +1,111 @@ package model import ( + "fmt" "time" -) -// TimeoutMode enum type -type TimeoutMode int + "github.com/rs/zerolog" -const ( - // ReplicaTimeout represents the time period that the replica is waiting for the block for the current view. - ReplicaTimeout TimeoutMode = iota - // VoteCollectionTimeout represents the time period that the leader is waiting for votes in order to build - // the next block. - VoteCollectionTimeout + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" ) -// TimerInfo represents a time period that pacemaker is waiting for a specific event. -// The end of the time period is the timeout that will trigger pacemaker's view change. +// TimerInfo represents a local timeout for a view, and indicates the Pacemaker has not yet +// observed evidence to transition to the next view (QC or TC). When a timeout occurs for +// the first time in a view, we will broadcast a TimeoutObject and continue waiting for evidence +// to enter the next view, but we will no longer submit a vote for this view. A timeout may occur +// multiple times for the same round, which is an indication +// to re-broadcast our TimeoutObject for the view, to ensure liveness. type TimerInfo struct { - Mode TimeoutMode - View uint64 + // View is round at which timer was created. + View uint64 + // StartTime represents time of entering the view. StartTime time.Time - Duration time.Duration + // Duration is how long we waited before timing out the view. + // It does not include subsequent timeouts (ie. all timeout events emitted for the same + // view will have the same Duration). + Duration time.Duration } -func (m TimeoutMode) String() string { - return [...]string{"ReplicaTimeout", "VoteCollectionTimeout"}[m] +// NewViewEvent indicates that a new view has started. While it has the same +// data model as `TimerInfo`, their semantics are different (hence we use +// different types): `TimerInfo` represents a continuous time interval. In +// contrast, NewViewEvent marks the specific point in time, when the timer +// is started. +type NewViewEvent TimerInfo + +// TimeoutObject represents intent of replica to leave its current view with a timeout. This concept is very similar to +// HotStuff vote. Valid TimeoutObject is signed by staking key. +type TimeoutObject struct { + // View is the view number which is replica is timing out + View uint64 + // NewestQC is the newest QC (by view) known to the creator of this TimeoutObject + NewestQC *flow.QuorumCertificate + // LastViewTC is the timeout certificate for previous view if NewestQC.View != View - 1, else nil + LastViewTC *flow.TimeoutCertificate + // SignerID is the identifier of replica that has signed this TimeoutObject + // SignerID must be the origin ID from networking layer, which cryptographically + // authenticates the message's sender. + SignerID flow.Identifier + // SigData is a BLS signature created by staking key signing View + NewestQC.View + // This signature is further aggregated in TimeoutCertificate. + SigData crypto.Signature + // TimeoutTick is the number of times the `timeout.Controller` has (re-)emitted the + // timeout for this view. When the timer for the view's original duration expires, a `TimeoutObject` + // with `TimeoutTick = 0` is broadcast. Subsequently, `timeout.Controller` re-broadcasts the + // `TimeoutObject` periodically based on some internal heuristic. Each time we attempt a re-broadcast, + // the `TimeoutTick` is incremented. Incrementing the field prevents de-duplicated within the network layer, + // which in turn guarantees quick delivery of the `TimeoutObject` after GST and facilitates recovery. + // This field is not part of timeout object ID. Thereby, two timeouts are identical if only they differ + // by their TimeoutTick value. + TimeoutTick uint64 +} + +// ID returns the TimeoutObject's identifier +func (t *TimeoutObject) ID() flow.Identifier { + body := struct { + View uint64 + NewestQCID flow.Identifier + LastViewTCID flow.Identifier + SignerID flow.Identifier + SigData crypto.Signature + }{ + View: t.View, + NewestQCID: t.NewestQC.ID(), + LastViewTCID: t.LastViewTC.ID(), + SignerID: t.SignerID, + SigData: t.SigData, + } + return flow.MakeID(body) +} + +func (t *TimeoutObject) String() string { + return fmt.Sprintf( + "View: %d, HighestQC.View: %d, LastViewTC: %v, TimeoutTick: %d", + t.View, + t.NewestQC.View, + t.LastViewTC, + t.TimeoutTick, + ) +} + +// LogContext returns a `zerolog.Contex` including the most important properties of the TC: +// - view number that this TC is for +// - view and ID of the block that the included QC points to +// - number of times a re-broadcast of this timeout was attempted +// - [optional] if the TC also includes a TC for the prior view, i.e. `LastViewTC` ≠ nil: +// the new of `LastViewTC` and the view that `LastViewTC.NewestQC` is for +func (t *TimeoutObject) LogContext(logger zerolog.Logger) zerolog.Context { + logContext := logger.With(). + Uint64("timeout_newest_qc_view", t.NewestQC.View). + Hex("timeout_newest_qc_block_id", t.NewestQC.BlockID[:]). + Uint64("timeout_view", t.View). + Uint64("timeout_tick", t.TimeoutTick) + + if t.LastViewTC != nil { + logContext. + Uint64("last_view_tc_view", t.LastViewTC.View). + Uint64("last_view_tc_newest_qc_view", t.LastViewTC.NewestQC.View) + } + return logContext } diff --git a/consensus/hotstuff/notifications/log_consumer.go b/consensus/hotstuff/notifications/log_consumer.go index c2653573dbd..0f3329c356d 100644 --- a/consensus/hotstuff/notifications/log_consumer.go +++ b/consensus/hotstuff/notifications/log_consumer.go @@ -1,6 +1,8 @@ package notifications import ( + "time" + "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" @@ -16,6 +18,7 @@ type LogConsumer struct { } var _ hotstuff.Consumer = (*LogConsumer)(nil) +var _ hotstuff.TimeoutCollectorConsumer = (*LogConsumer)(nil) func NewLogConsumer(log zerolog.Logger) *LogConsumer { lc := &LogConsumer{ @@ -28,6 +31,10 @@ func (lc *LogConsumer) OnEventProcessed() { lc.log.Debug().Msg("event processed") } +func (lc *LogConsumer) OnStart(currentView uint64) { + lc.log.Debug().Uint64("cur_view", currentView).Msg("starting event handler") +} + func (lc *LogConsumer) OnBlockIncorporated(block *model.Block) { lc.logBasicBlockData(lc.log.Debug(), block). Msg("block incorporated") @@ -47,85 +54,113 @@ func (lc *LogConsumer) OnDoubleProposeDetected(block *model.Block, alt *model.Bl Msg("double proposal detected") } -func (lc *LogConsumer) OnReceiveVote(currentView uint64, vote *model.Vote) { +func (lc *LogConsumer) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { + logger := lc.logBasicBlockData(lc.log.Debug(), proposal.Block). + Uint64("cur_view", currentView) + lastViewTC := proposal.LastViewTC + if lastViewTC != nil { + logger. + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) + } + + logger.Msg("processing proposal") +} + +func (lc *LogConsumer) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { lc.log.Debug(). Uint64("cur_view", currentView). - Uint64("vote_view", vote.View). - Hex("voted_block_id", vote.BlockID[:]). - Hex("voter_id", vote.SignerID[:]). - Msg("processing vote") + Uint64("qc_view", qc.View). + Hex("qc_block_id", logging.ID(qc.BlockID)). + Msg("processing QC") } -func (lc *LogConsumer) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { - lc.logBasicBlockData(lc.log.Debug(), proposal.Block). +func (lc *LogConsumer) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + lc.log.Debug(). Uint64("cur_view", currentView). - Msg("processing proposal") + Uint64("tc_view", tc.View). + Uint64("newest_qc_view", tc.NewestQC.View). + Hex("newest_qc_block_id", logging.ID(tc.NewestQC.BlockID)). + Msg("processing TC") } -func (lc *LogConsumer) OnEnteringView(view uint64, leader flow.Identifier) { - lc.log.Debug(). - Uint64("view", view). - Hex("leader", leader[:]). - Msg("view entered") +func (lc *LogConsumer) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + logger := lc.log.With(). + Uint64("cur_view", currentView). + Uint64("view", partialTc.View). + Uint64("qc_view", partialTc.NewestQC.View). + Hex("qc_block_id", logging.ID(partialTc.NewestQC.BlockID)) + + lastViewTC := partialTc.LastViewTC + if lastViewTC != nil { + logger. + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) + } + + log := logger.Logger() + log.Debug().Msg("processing partial TC") } -func (lc *LogConsumer) OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) { +func (lc *LogConsumer) OnLocalTimeout(currentView uint64) { lc.log.Debug(). - Uint64("qc_view", qc.View). - Hex("qc_id", qc.BlockID[:]). - Uint64("new_view", newView). - Msg("QC triggered view change") + Uint64("cur_view", currentView). + Msg("processing local timeout") } -func (lc *LogConsumer) OnProposingBlock(block *model.Proposal) { - lc.logBasicBlockData(lc.log.Debug(), block.Block). - Msg("proposing block") +func (lc *LogConsumer) OnViewChange(oldView, newView uint64) { + lc.log.Debug(). + Uint64("old_view", oldView). + Uint64("new_view", newView). + Msg("entered new view") } -func (lc *LogConsumer) OnVoting(vote *model.Vote) { +func (lc *LogConsumer) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { lc.log.Debug(). - Uint64("block_view", vote.View). - Hex("block_id", vote.BlockID[:]). - Msg("voting for block") + Uint64("qc_view", qc.View). + Hex("qc_block_id", qc.BlockID[:]). + Uint64("old_view", oldView). + Uint64("new_view", newView). + Msg("QC triggered view change") } -func (lc *LogConsumer) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) { +func (lc *LogConsumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { lc.log.Debug(). - Uint64("cur_view", curView). - Uint64("qc_view", qc.View). - Hex("qc_id", qc.BlockID[:]). - Msg("QC constructed from votes") + Uint64("tc_view", tc.View). + Uint64("tc_newest_qc_view", tc.NewestQC.View). + Uint64("new_view", newView). + Uint64("old_view", oldView). + Msg("TC triggered view change") } -func (lc *LogConsumer) OnStartingTimeout(info *model.TimerInfo) { +func (lc *LogConsumer) OnStartingTimeout(info model.TimerInfo) { lc.log.Debug(). Uint64("timeout_view", info.View). Time("timeout_cutoff", info.StartTime.Add(info.Duration)). - Str("timeout_mode", info.Mode.String()). Msg("timeout started") } -func (lc *LogConsumer) OnReachedTimeout(info *model.TimerInfo) { +func (lc *LogConsumer) OnVoteProcessed(vote *model.Vote) { lc.log.Debug(). - Uint64("timeout_view", info.View). - Time("timeout_cutoff", info.StartTime.Add(info.Duration)). - Str("timeout_mode", info.Mode.String()). - Msg("timeout reached") + Hex("block_id", vote.BlockID[:]). + Uint64("block_view", vote.View). + Hex("recipient_id", vote.SignerID[:]). + Msg("processed valid HotStuff vote") } -func (lc *LogConsumer) OnQcIncorporated(qc *flow.QuorumCertificate) { - lc.log.Debug(). - Uint64("qc_view", qc.View). - Hex("qc_id", qc.BlockID[:]). - Msg("QC incorporated") +func (lc *LogConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + log := timeout.LogContext(lc.log).Logger() + log.Debug().Msg("processed valid timeout object") } -func (lc *LogConsumer) OnForkChoiceGenerated(view uint64, qc *flow.QuorumCertificate) { - lc.log.Debug(). - Uint64("proposal_view", view). - Uint64("qc_view", qc.View). - Hex("qc_id", qc.BlockID[:]). - Msg("fork choice generated") +func (lc *LogConsumer) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { + lc.log.Info(). + Uint64("view", currentView). + Uint64("finalized_view", finalizedView). + Hex("current_leader", currentLeader[:]). + Msg("current view details") } func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) { @@ -137,12 +172,12 @@ func (lc *LogConsumer) OnDoubleVotingDetected(vote *model.Vote, alt *model.Vote) Msg("double vote detected") } -func (lc *LogConsumer) OnInvalidVoteDetected(vote *model.Vote) { +func (lc *LogConsumer) OnInvalidVoteDetected(err model.InvalidVoteError) { lc.log.Warn(). - Uint64("vote_view", vote.View). - Hex("voted_block_id", vote.BlockID[:]). - Hex("voter_id", vote.SignerID[:]). - Msg("invalid vote detected") + Uint64("vote_view", err.Vote.View). + Hex("voted_block_id", err.Vote.BlockID[:]). + Hex("voter_id", err.Vote.SignerID[:]). + Msgf("invalid vote detected: %s", err.Error()) } func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal *model.Proposal) { @@ -154,16 +189,87 @@ func (lc *LogConsumer) OnVoteForInvalidBlockDetected(vote *model.Vote, proposal Msg("vote for invalid proposal detected") } +func (lc *LogConsumer) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, alt *model.TimeoutObject) { + lc.log.Warn(). + Uint64("timeout_view", timeout.View). + Hex("signer_id", logging.ID(timeout.SignerID)). + Hex("timeout_id", logging.ID(timeout.ID())). + Hex("alt_id", logging.ID(alt.ID())). + Msg("double timeout detected") +} + +func (lc *LogConsumer) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + log := err.Timeout.LogContext(lc.log).Logger() + log.Warn().Msgf("invalid timeout detected: %s", err.Error()) +} + func (lc *LogConsumer) logBasicBlockData(loggerEvent *zerolog.Event, block *model.Block) *zerolog.Event { loggerEvent. Uint64("block_view", block.View). Hex("block_id", logging.ID(block.BlockID)). Hex("proposer_id", logging.ID(block.ProposerID)). - Hex("payload_hash", logging.ID(block.PayloadHash)) - if block.QC != nil { - loggerEvent. - Uint64("qc_view", block.QC.View). - Hex("qc_id", logging.ID(block.QC.BlockID)) - } + Hex("payload_hash", logging.ID(block.PayloadHash)). + Uint64("qc_view", block.QC.View). + Hex("qc_block_id", logging.ID(block.QC.BlockID)) + return loggerEvent } + +func (lc *LogConsumer) OnTcConstructedFromTimeouts(tc *flow.TimeoutCertificate) { + lc.log.Debug(). + Uint64("tc_view", tc.View). + Uint64("newest_qc_view", tc.NewestQC.View). + Hex("newest_qc_block_id", tc.NewestQC.BlockID[:]). + Msg("TC constructed") +} + +func (lc *LogConsumer) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + lc.log.Debug(). + Uint64("view", view). + Uint64("newest_qc_view", newestQC.View). + Hex("newest_qc_block_id", newestQC.BlockID[:]). + Bool("has_last_view_tc", lastViewTC != nil). + Msg("partial TC constructed") +} + +func (lc *LogConsumer) OnNewQcDiscovered(qc *flow.QuorumCertificate) { + lc.log.Debug(). + Uint64("qc_view", qc.View). + Hex("qc_block_id", qc.BlockID[:]). + Msg("new QC discovered") +} + +func (lc *LogConsumer) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { + lc.log.Debug(). + Uint64("tc_view", tc.View). + Uint64("newest_qc_view", tc.NewestQC.View). + Hex("newest_qc_block_id", tc.NewestQC.BlockID[:]). + Msg("new TC discovered") +} + +func (lc *LogConsumer) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + lc.log.Debug(). + Hex("block_id", blockID[:]). + Uint64("block_view", view). + Hex("recipient_id", recipientID[:]). + Msg("publishing HotStuff vote") +} + +func (lc *LogConsumer) OnOwnTimeout(timeout *model.TimeoutObject) { + log := timeout.LogContext(lc.log).Logger() + log.Debug().Msg("publishing HotStuff timeout object") +} + +func (lc *LogConsumer) OnOwnProposal(header *flow.Header, targetPublicationTime time.Time) { + lc.log.Debug(). + Str("chain_id", header.ChainID.String()). + Uint64("block_height", header.Height). + Uint64("block_view", header.View). + Hex("block_id", logging.Entity(header)). + Hex("parent_id", header.ParentID[:]). + Hex("payload_hash", header.PayloadHash[:]). + Time("timestamp", header.Timestamp). + Hex("parent_signer_indices", header.ParentVoterIndices). + Time("target_publication_time", targetPublicationTime). + Msg("publishing HotStuff block proposal") +} diff --git a/consensus/hotstuff/notifications/noop_consumer.go b/consensus/hotstuff/notifications/noop_consumer.go index 392362db375..b5d980acdd3 100644 --- a/consensus/hotstuff/notifications/noop_consumer.go +++ b/consensus/hotstuff/notifications/noop_consumer.go @@ -1,6 +1,8 @@ package notifications import ( + "time" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" @@ -8,7 +10,11 @@ import ( // NoopConsumer is an implementation of the notifications consumer that // doesn't do anything. -type NoopConsumer struct{} +type NoopConsumer struct { + NoopFinalizationConsumer + NoopPartialConsumer + NoopCommunicatorConsumer +} var _ hotstuff.Consumer = (*NoopConsumer)(nil) @@ -17,38 +23,91 @@ func NewNoopConsumer() *NoopConsumer { return nc } -func (*NoopConsumer) OnEventProcessed() {} +// no-op implementation of hotstuff.Consumer(but not nested interfaces) + +type NoopPartialConsumer struct{} + +func (*NoopPartialConsumer) OnEventProcessed() {} + +func (*NoopPartialConsumer) OnStart(uint64) {} + +func (*NoopPartialConsumer) OnReceiveProposal(uint64, *model.Proposal) {} + +func (*NoopPartialConsumer) OnReceiveQc(uint64, *flow.QuorumCertificate) {} + +func (*NoopPartialConsumer) OnReceiveTc(uint64, *flow.TimeoutCertificate) {} + +func (*NoopPartialConsumer) OnPartialTc(uint64, *hotstuff.PartialTcCreated) {} + +func (*NoopPartialConsumer) OnLocalTimeout(uint64) {} + +func (*NoopPartialConsumer) OnViewChange(uint64, uint64) {} + +func (*NoopPartialConsumer) OnQcTriggeredViewChange(uint64, uint64, *flow.QuorumCertificate) {} + +func (*NoopPartialConsumer) OnTcTriggeredViewChange(uint64, uint64, *flow.TimeoutCertificate) {} + +func (*NoopPartialConsumer) OnStartingTimeout(model.TimerInfo) {} + +func (*NoopPartialConsumer) OnVoteProcessed(*model.Vote) {} -func (*NoopConsumer) OnBlockIncorporated(*model.Block) {} +func (*NoopPartialConsumer) OnTimeoutProcessed(*model.TimeoutObject) {} -func (*NoopConsumer) OnFinalizedBlock(*model.Block) {} +func (*NoopPartialConsumer) OnCurrentViewDetails(uint64, uint64, flow.Identifier) {} -func (*NoopConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} +func (*NoopPartialConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} -func (c *NoopConsumer) OnReceiveVote(uint64, *model.Vote) {} +func (*NoopPartialConsumer) OnInvalidVoteDetected(model.InvalidVoteError) {} -func (c *NoopConsumer) OnReceiveProposal(uint64, *model.Proposal) {} +func (*NoopPartialConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} + +func (*NoopPartialConsumer) OnDoubleTimeoutDetected(*model.TimeoutObject, *model.TimeoutObject) {} + +func (*NoopPartialConsumer) OnInvalidTimeoutDetected(model.InvalidTimeoutError) {} + +// no-op implementation of hotstuff.FinalizationConsumer + +type NoopFinalizationConsumer struct{} + +var _ hotstuff.FinalizationConsumer = (*NoopFinalizationConsumer)(nil) + +func (*NoopFinalizationConsumer) OnBlockIncorporated(*model.Block) {} + +func (*NoopFinalizationConsumer) OnFinalizedBlock(*model.Block) {} + +func (*NoopFinalizationConsumer) OnDoubleProposeDetected(*model.Block, *model.Block) {} + +// no-op implementation of hotstuff.TimeoutCollectorConsumer + +type NoopTimeoutCollectorConsumer struct{} + +var _ hotstuff.TimeoutCollectorConsumer = (*NoopTimeoutCollectorConsumer)(nil) + +func (*NoopTimeoutCollectorConsumer) OnTcConstructedFromTimeouts(*flow.TimeoutCertificate) {} + +func (*NoopTimeoutCollectorConsumer) OnPartialTcCreated(uint64, *flow.QuorumCertificate, *flow.TimeoutCertificate) { +} -func (*NoopConsumer) OnEnteringView(uint64, flow.Identifier) {} +func (*NoopTimeoutCollectorConsumer) OnNewQcDiscovered(*flow.QuorumCertificate) {} -func (c *NoopConsumer) OnQcTriggeredViewChange(*flow.QuorumCertificate, uint64) {} +func (*NoopTimeoutCollectorConsumer) OnNewTcDiscovered(*flow.TimeoutCertificate) {} -func (c *NoopConsumer) OnProposingBlock(*model.Proposal) {} +// no-op implementation of hotstuff.CommunicatorConsumer -func (c *NoopConsumer) OnVoting(*model.Vote) {} +type NoopCommunicatorConsumer struct{} -func (c *NoopConsumer) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) {} +var _ hotstuff.CommunicatorConsumer = (*NoopCommunicatorConsumer)(nil) -func (*NoopConsumer) OnStartingTimeout(*model.TimerInfo) {} +func (*NoopCommunicatorConsumer) OnOwnVote(flow.Identifier, uint64, []byte, flow.Identifier) {} -func (*NoopConsumer) OnReachedTimeout(*model.TimerInfo) {} +func (*NoopCommunicatorConsumer) OnOwnTimeout(*model.TimeoutObject) {} -func (*NoopConsumer) OnQcIncorporated(*flow.QuorumCertificate) {} +func (*NoopCommunicatorConsumer) OnOwnProposal(*flow.Header, time.Time) {} -func (*NoopConsumer) OnForkChoiceGenerated(uint64, *flow.QuorumCertificate) {} +// no-op implementation of hotstuff.QCCreatedConsumer -func (*NoopConsumer) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} +type NoopQCCreatedConsumer struct{} -func (*NoopConsumer) OnInvalidVoteDetected(*model.Vote) {} +var _ hotstuff.QCCreatedConsumer = (*NoopQCCreatedConsumer)(nil) -func (*NoopConsumer) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} +func (*NoopQCCreatedConsumer) OnQcConstructedFromVotes(*flow.QuorumCertificate) {} diff --git a/consensus/hotstuff/notifications/pubsub/distributor.go b/consensus/hotstuff/notifications/pubsub/distributor.go index 81371875e12..d122ad8cde3 100644 --- a/consensus/hotstuff/notifications/pubsub/distributor.go +++ b/consensus/hotstuff/notifications/pubsub/distributor.go @@ -2,6 +2,7 @@ package pubsub import ( "sync" + "time" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" @@ -30,18 +31,18 @@ func NewDistributor() *Distributor { return &Distributor{} } -// AddConsumer adds an a event consumer to the Distributor +// AddConsumer adds an event consumer to the Distributor func (p *Distributor) AddConsumer(consumer hotstuff.Consumer) { p.lock.Lock() defer p.lock.Unlock() p.subscribers = append(p.subscribers, consumer) } -func (p *Distributor) OnReceiveVote(currentView uint64, vote *model.Vote) { +func (p *Distributor) OnStart(currentView uint64) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnReceiveVote(currentView, vote) + subscriber.OnStart(currentView) } } @@ -53,47 +54,63 @@ func (p *Distributor) OnReceiveProposal(currentView uint64, proposal *model.Prop } } -func (p *Distributor) OnEnteringView(view uint64, leader flow.Identifier) { +func (p *Distributor) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnEnteringView(view, leader) + subscriber.OnReceiveQc(currentView, qc) } } -func (p *Distributor) OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) { +func (p *Distributor) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnQcTriggeredViewChange(qc, newView) + subscriber.OnReceiveTc(currentView, tc) } } -func (p *Distributor) OnProposingBlock(proposal *model.Proposal) { +func (p *Distributor) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnProposingBlock(proposal) + subscriber.OnPartialTc(currentView, partialTc) } } -func (p *Distributor) OnVoting(vote *model.Vote) { +func (p *Distributor) OnLocalTimeout(currentView uint64) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnVoting(vote) + subscriber.OnLocalTimeout(currentView) } } -func (p *Distributor) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) { +func (p *Distributor) OnViewChange(oldView, newView uint64) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnQcConstructedFromVotes(curView, qc) + subscriber.OnViewChange(oldView, newView) } } -func (p *Distributor) OnStartingTimeout(timerInfo *model.TimerInfo) { +func (p *Distributor) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnQcTriggeredViewChange(oldView, newView, qc) + } +} + +func (p *Distributor) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnTcTriggeredViewChange(oldView, newView, tc) + } +} + +func (p *Distributor) OnStartingTimeout(timerInfo model.TimerInfo) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { @@ -101,27 +118,27 @@ func (p *Distributor) OnStartingTimeout(timerInfo *model.TimerInfo) { } } -func (p *Distributor) OnReachedTimeout(timeout *model.TimerInfo) { +func (p *Distributor) OnVoteProcessed(vote *model.Vote) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnReachedTimeout(timeout) + subscriber.OnVoteProcessed(vote) } } -func (p *Distributor) OnQcIncorporated(qc *flow.QuorumCertificate) { +func (p *Distributor) OnTimeoutProcessed(timeout *model.TimeoutObject) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnQcIncorporated(qc) + subscriber.OnTimeoutProcessed(timeout) } } -func (p *Distributor) OnForkChoiceGenerated(curView uint64, selectedQC *flow.QuorumCertificate) { +func (p *Distributor) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnForkChoiceGenerated(curView, selectedQC) + subscriber.OnCurrentViewDetails(currentView, finalizedView, currentLeader) } } @@ -157,11 +174,11 @@ func (p *Distributor) OnDoubleVotingDetected(vote1, vote2 *model.Vote) { } } -func (p *Distributor) OnInvalidVoteDetected(vote *model.Vote) { +func (p *Distributor) OnInvalidVoteDetected(err model.InvalidVoteError) { p.lock.RLock() defer p.lock.RUnlock() for _, subscriber := range p.subscribers { - subscriber.OnInvalidVoteDetected(vote) + subscriber.OnInvalidVoteDetected(err) } } @@ -172,3 +189,43 @@ func (p *Distributor) OnVoteForInvalidBlockDetected(vote *model.Vote, invalidPro subscriber.OnVoteForInvalidBlockDetected(vote, invalidProposal) } } + +func (p *Distributor) OnDoubleTimeoutDetected(timeout *model.TimeoutObject, altTimeout *model.TimeoutObject) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnDoubleTimeoutDetected(timeout, altTimeout) + } +} + +func (p *Distributor) OnInvalidTimeoutDetected(err model.InvalidTimeoutError) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, subscriber := range p.subscribers { + subscriber.OnInvalidTimeoutDetected(err) + } +} + +func (p *Distributor) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnVote(blockID, view, sigData, recipientID) + } +} + +func (p *Distributor) OnOwnTimeout(timeout *model.TimeoutObject) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnTimeout(timeout) + } +} + +func (p *Distributor) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + p.lock.RLock() + defer p.lock.RUnlock() + for _, s := range p.subscribers { + s.OnOwnProposal(proposal, targetPublicationTime) + } +} diff --git a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go index 1c571ab4030..6d1c72ef8e6 100644 --- a/consensus/hotstuff/notifications/pubsub/finalization_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/finalization_distributor.go @@ -5,14 +5,15 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" ) type OnBlockFinalizedConsumer = func(block *model.Block) type OnBlockIncorporatedConsumer = func(block *model.Block) -// FinalizationDistributor subscribes for finalization events from hotstuff and distributes it to subscribers +// FinalizationDistributor ingests finalization events from hotstuff and distributes it to subscribers. type FinalizationDistributor struct { + notifications.NoopConsumer blockFinalizedConsumers []OnBlockFinalizedConsumer blockIncorporatedConsumers []OnBlockIncorporatedConsumer hotStuffFinalizationConsumers []hotstuff.FinalizationConsumer @@ -47,8 +48,6 @@ func (p *FinalizationDistributor) AddConsumer(consumer hotstuff.FinalizationCons p.hotStuffFinalizationConsumers = append(p.hotStuffFinalizationConsumers, consumer) } -func (p *FinalizationDistributor) OnEventProcessed() {} - func (p *FinalizationDistributor) OnBlockIncorporated(block *model.Block) { p.lock.RLock() defer p.lock.RUnlock() @@ -78,32 +77,3 @@ func (p *FinalizationDistributor) OnDoubleProposeDetected(block1, block2 *model. consumer.OnDoubleProposeDetected(block1, block2) } } - -func (p *FinalizationDistributor) OnReceiveVote(uint64, *model.Vote) {} - -func (p *FinalizationDistributor) OnReceiveProposal(uint64, *model.Proposal) {} - -func (p *FinalizationDistributor) OnEnteringView(uint64, flow.Identifier) {} - -func (p *FinalizationDistributor) OnQcTriggeredViewChange(*flow.QuorumCertificate, uint64) {} - -func (p *FinalizationDistributor) OnProposingBlock(*model.Proposal) {} - -func (p *FinalizationDistributor) OnVoting(*model.Vote) {} - -func (p *FinalizationDistributor) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) { -} - -func (p *FinalizationDistributor) OnStartingTimeout(*model.TimerInfo) {} - -func (p *FinalizationDistributor) OnReachedTimeout(*model.TimerInfo) {} - -func (p *FinalizationDistributor) OnQcIncorporated(*flow.QuorumCertificate) {} - -func (p *FinalizationDistributor) OnForkChoiceGenerated(uint64, *flow.QuorumCertificate) {} - -func (p *FinalizationDistributor) OnDoubleVotingDetected(*model.Vote, *model.Vote) {} - -func (p *FinalizationDistributor) OnInvalidVoteDetected(*model.Vote) {} - -func (p *FinalizationDistributor) OnVoteForInvalidBlockDetected(*model.Vote, *model.Proposal) {} diff --git a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go index 7a2df781e91..166fa9cf757 100644 --- a/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go +++ b/consensus/hotstuff/notifications/pubsub/qc_created_distributor.go @@ -7,14 +7,12 @@ import ( "github.com/onflow/flow-go/model/flow" ) -type OnQCCreatedConsumer = func(qc *flow.QuorumCertificate) - -// QCCreatedDistributor subscribes for qc created event from hotstuff and distributes it to subscribers +// QCCreatedDistributor ingests events about QC creation from hotstuff and distributes them to subscribers. // Objects are concurrency safe. // NOTE: it can be refactored to work without lock since usually we never subscribe after startup. Mostly // list of observers is static. type QCCreatedDistributor struct { - qcCreatedConsumers []OnQCCreatedConsumer + qcCreatedConsumers []hotstuff.QCCreatedConsumer lock sync.RWMutex } @@ -22,11 +20,11 @@ var _ hotstuff.QCCreatedConsumer = (*QCCreatedDistributor)(nil) func NewQCCreatedDistributor() *QCCreatedDistributor { return &QCCreatedDistributor{ - qcCreatedConsumers: make([]OnQCCreatedConsumer, 0), + qcCreatedConsumers: make([]hotstuff.QCCreatedConsumer, 0), } } -func (d *QCCreatedDistributor) AddConsumer(consumer OnQCCreatedConsumer) { +func (d *QCCreatedDistributor) AddConsumer(consumer hotstuff.QCCreatedConsumer) { d.lock.Lock() defer d.lock.Unlock() d.qcCreatedConsumers = append(d.qcCreatedConsumers, consumer) @@ -36,6 +34,6 @@ func (d *QCCreatedDistributor) OnQcConstructedFromVotes(qc *flow.QuorumCertifica d.lock.RLock() defer d.lock.RUnlock() for _, consumer := range d.qcCreatedConsumers { - consumer(qc) + consumer.OnQcConstructedFromVotes(qc) } } diff --git a/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go new file mode 100644 index 00000000000..8387fb81663 --- /dev/null +++ b/consensus/hotstuff/notifications/pubsub/timeout_collector_distributor.go @@ -0,0 +1,63 @@ +package pubsub + +import ( + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/model/flow" +) + +// TimeoutCollectorDistributor ingests events from hotstuff and distributes them to subscribers. +// Concurrently safe +// TODO: investigate if this can be updated using atomics to prevent locking on mutex since we always add all consumers +// before delivering events. +type TimeoutCollectorDistributor struct { + lock sync.RWMutex + consumers []hotstuff.TimeoutCollectorConsumer +} + +var _ hotstuff.TimeoutCollectorConsumer = (*TimeoutCollectorDistributor)(nil) + +func NewTimeoutCollectorDistributor() *TimeoutCollectorDistributor { + return &TimeoutCollectorDistributor{ + consumers: make([]hotstuff.TimeoutCollectorConsumer, 0), + } +} + +func (d *TimeoutCollectorDistributor) AddConsumer(consumer hotstuff.TimeoutCollectorConsumer) { + d.lock.Lock() + defer d.lock.Unlock() + d.consumers = append(d.consumers, consumer) +} + +func (d *TimeoutCollectorDistributor) OnTcConstructedFromTimeouts(tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnTcConstructedFromTimeouts(tc) + } +} + +func (d *TimeoutCollectorDistributor) OnPartialTcCreated(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnPartialTcCreated(view, newestQC, lastViewTC) + } +} + +func (d *TimeoutCollectorDistributor) OnNewQcDiscovered(qc *flow.QuorumCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnNewQcDiscovered(qc) + } +} + +func (d *TimeoutCollectorDistributor) OnNewTcDiscovered(tc *flow.TimeoutCertificate) { + d.lock.RLock() + defer d.lock.RUnlock() + for _, consumer := range d.consumers { + consumer.OnNewTcDiscovered(tc) + } +} diff --git a/consensus/hotstuff/notifications/telemetry.go b/consensus/hotstuff/notifications/telemetry.go index 5a64f3f4ce7..67f0ca1339a 100644 --- a/consensus/hotstuff/notifications/telemetry.go +++ b/consensus/hotstuff/notifications/telemetry.go @@ -1,73 +1,121 @@ package notifications import ( + "time" + "github.com/google/uuid" "github.com/rs/zerolog" - "github.com/onflow/flow-go/utils/logging" - "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/logging" ) // TelemetryConsumer implements the hotstuff.Consumer interface. -// consumes outbound notifications produced by the The HotStuff state machine. +// consumes outbound notifications produced by the HotStuff state machine. // For this purpose, the TelemetryConsumer enriches the state machine's notifications: // - The goal is to identify all events as belonging together that were emitted during // a path through the state machine. // - A path through the state machine begins when: -// -- a vote is received -// -- a block is received -// -- a new view is started -// -- a timeout is processed +// -- a block has been received +// -- a QC has been constructed +// -- a TC has been constructed +// -- a partial TC has been constructed +// -- a local timeout has been initiated // - Each path through the state machine is identified by a unique id. // +// Additionally, the TelemetryConsumer reports events related to vote and timeout aggregation +// but those events are not bound to a path, so they are reported differently. // Generally, the TelemetryConsumer could export the collected data to a variety of backends. // For now, we export the data to a logger. // // Telemetry does NOT capture slashing notifications type TelemetryConsumer struct { NoopConsumer - pathHandler *PathHandler + pathHandler *PathHandler + noPathLogger zerolog.Logger } var _ hotstuff.Consumer = (*TelemetryConsumer)(nil) -func NewTelemetryConsumer(log zerolog.Logger, chain flow.ChainID) *TelemetryConsumer { +// NewTelemetryConsumer creates consumer that reports telemetry events using logger backend. +// Logger MUST include `chain` parameter as part of log context with corresponding chain ID to correctly map telemetry events to chain. +func NewTelemetryConsumer(log zerolog.Logger) *TelemetryConsumer { + pathHandler := NewPathHandler(log) return &TelemetryConsumer{ - pathHandler: NewPathHandler(log, chain), + pathHandler: pathHandler, + noPathLogger: pathHandler.log, } } -func (t *TelemetryConsumer) OnReceiveVote(currentView uint64, vote *model.Vote) { - // TODO: update - // As of Consensus Voting V2, receiving a vote is not an event within the HotStuff state machine anymore. - //t.pathHandler.StartNextPath(currentView) - // t.pathHandler.NextStep(). - // Uint64("voted_block_view", vote.View). - // Hex("voted_block_id", vote.BlockID[:]). - // Hex("voter_id", vote.SignerID[:]). - // Msg("OnReceiveVote") +func (t *TelemetryConsumer) OnStart(currentView uint64) { + t.pathHandler.StartNextPath(currentView) + t.pathHandler.NextStep().Msg("OnStart") } func (t *TelemetryConsumer) OnReceiveProposal(currentView uint64, proposal *model.Proposal) { block := proposal.Block t.pathHandler.StartNextPath(currentView) - step := t.pathHandler.NextStep() - step. + step := t.pathHandler.NextStep(). Uint64("block_view", block.View). Hex("block_id", logging.ID(block.BlockID)). Hex("block_proposer_id", logging.ID(block.ProposerID)). - Time("block_time", block.Timestamp) - if block.QC != nil { + Time("block_time", block.Timestamp). + Uint64("qc_view", block.QC.View). + Hex("qc_block_id", logging.ID(block.QC.BlockID)) + + lastViewTC := proposal.LastViewTC + if lastViewTC != nil { step. - Uint64("qc_block_view", block.QC.View). - Hex("qc_block_id", logging.ID(block.QC.BlockID)) + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) } + step.Msg("OnReceiveProposal") } +func (t *TelemetryConsumer) OnReceiveQc(currentView uint64, qc *flow.QuorumCertificate) { + t.pathHandler.StartNextPath(currentView) + t.pathHandler.NextStep(). + Uint64("qc_view", qc.View). + Hex("qc_block_id", logging.ID(qc.BlockID)). + Msg("OnReceiveQc") +} + +func (t *TelemetryConsumer) OnReceiveTc(currentView uint64, tc *flow.TimeoutCertificate) { + t.pathHandler.StartNextPath(currentView) + t.pathHandler.NextStep(). + Uint64("view", tc.View). + Uint64("newest_qc_view", tc.NewestQC.View). + Hex("newest_qc_block_id", logging.ID(tc.NewestQC.BlockID)). + Msg("OnReceiveTc") +} + +func (t *TelemetryConsumer) OnPartialTc(currentView uint64, partialTc *hotstuff.PartialTcCreated) { + t.pathHandler.StartNextPath(currentView) + step := t.pathHandler.NextStep(). + Uint64("view", partialTc.View). + Uint64("newest_qc_view", partialTc.NewestQC.View). + Hex("newest_qc_block_id", logging.ID(partialTc.NewestQC.BlockID)) + + lastViewTC := partialTc.LastViewTC + if lastViewTC != nil { + step. + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) + } + + step.Msg("OnPartialTc") +} + +func (t *TelemetryConsumer) OnLocalTimeout(currentView uint64) { + t.pathHandler.StartNextPath(currentView) + t.pathHandler.NextStep().Msg("OnLocalTimeout") +} + func (t *TelemetryConsumer) OnEventProcessed() { if t.pathHandler.IsCurrentPathClosed() { return @@ -77,34 +125,13 @@ func (t *TelemetryConsumer) OnEventProcessed() { t.pathHandler.CloseCurrentPath() } -func (t *TelemetryConsumer) OnStartingTimeout(info *model.TimerInfo) { - if info.Mode == model.ReplicaTimeout { - // the PaceMarker starts a new ReplicaTimeout if and only if it transitions to a higher view - t.pathHandler.StartNextPath(info.View) - } +func (t *TelemetryConsumer) OnStartingTimeout(info model.TimerInfo) { t.pathHandler.NextStep(). - Str("timeout_mode", info.Mode.String()). Float64("timeout_duration_seconds", info.Duration.Seconds()). Time("timeout_cutoff", info.StartTime.Add(info.Duration)). Msg("OnStartingTimeout") } -func (t *TelemetryConsumer) OnEnteringView(viewNumber uint64, leader flow.Identifier) { - t.pathHandler.NextStep(). - Uint64("entered_view", viewNumber). - Hex("leader", leader[:]). - Msg("OnEnteringView") -} - -func (t *TelemetryConsumer) OnReachedTimeout(info *model.TimerInfo) { - t.pathHandler.StartNextPath(info.View) - t.pathHandler.NextStep(). - Str("timeout_mode", info.Mode.String()). - Time("timeout_start_time", info.StartTime). - Float64("timeout_duration_seconds", info.Duration.Seconds()). - Msg("OnReachedTimeout") -} - func (t *TelemetryConsumer) OnBlockIncorporated(block *model.Block) { t.pathHandler.NextStep(). Hex("block_id", logging.ID(block.BlockID)). @@ -117,60 +144,100 @@ func (t *TelemetryConsumer) OnFinalizedBlock(block *model.Block) { Msg("OnFinalizedBlock") } -func (t *TelemetryConsumer) OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) { +func (t *TelemetryConsumer) OnQcTriggeredViewChange(oldView uint64, newView uint64, qc *flow.QuorumCertificate) { t.pathHandler.NextStep(). - Uint64("qc_block_view", qc.View). + Uint64("qc_view", qc.View). + Uint64("old_view", oldView). Uint64("next_view", newView). Hex("qc_block_id", qc.BlockID[:]). Msg("OnQcTriggeredViewChange") } -func (t *TelemetryConsumer) OnProposingBlock(proposal *model.Proposal) { - block := proposal.Block - step := t.pathHandler.NextStep() - step. - Uint64("block_view", block.View). - Hex("block_id", logging.ID(block.BlockID)). - Hex("block_proposer_id", logging.ID(block.ProposerID)). - Time("block_time", block.Timestamp) - if block.QC != nil { +func (t *TelemetryConsumer) OnTcTriggeredViewChange(oldView uint64, newView uint64, tc *flow.TimeoutCertificate) { + t.pathHandler.NextStep(). + Uint64("tc_view", tc.View). + Uint64("old_view", oldView). + Uint64("next_view", newView). + Uint64("tc_newest_qc_view", tc.NewestQC.View). + Hex("tc_newest_qc_block_id", tc.NewestQC.BlockID[:]). + Msg("OnTcTriggeredViewChange") +} + +func (t *TelemetryConsumer) OnOwnVote(blockID flow.Identifier, view uint64, _ []byte, recipientID flow.Identifier) { + t.pathHandler.NextStep(). + Uint64("voted_block_view", view). + Hex("voted_block_id", logging.ID(blockID)). + Hex("recipient_id", logging.ID(recipientID)). + Msg("OnOwnVote") +} + +func (t *TelemetryConsumer) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + step := t.pathHandler.NextStep(). + Uint64("block_view", proposal.View). + Hex("block_id", logging.ID(proposal.ID())). + Hex("block_proposer_id", logging.ID(proposal.ProposerID)). + Time("block_time", proposal.Timestamp). + Uint64("qc_view", proposal.ParentView). + Hex("qc_block_id", logging.ID(proposal.ParentID)). + Time("targetPublicationTime", targetPublicationTime) + lastViewTC := proposal.LastViewTC + if lastViewTC != nil { step. - Uint64("qc_block_view", block.QC.View). - Hex("qc_block_id", logging.ID(block.QC.BlockID)) + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) } - step.Msg("OnProposingBlock") + step.Msg("OnOwnProposal") } -func (t *TelemetryConsumer) OnVoting(vote *model.Vote) { - t.pathHandler.NextStep(). - Uint64("voted_block_view", vote.View). - Hex("voted_block_id", vote.BlockID[:]). - Hex("voter_id", vote.SignerID[:]). - Msg("OnVoting") +func (t *TelemetryConsumer) OnOwnTimeout(timeout *model.TimeoutObject) { + step := t.pathHandler.NextStep(). + Uint64("view", timeout.View). + Uint64("timeout_tick", timeout.TimeoutTick). + Uint64("newest_qc_view", timeout.NewestQC.View). + Hex("newest_qc_block_id", logging.ID(timeout.NewestQC.BlockID)) + + lastViewTC := timeout.LastViewTC + if lastViewTC != nil { + step. + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) + } + step.Msg("OnOwnTimeout") } -func (t *TelemetryConsumer) OnForkChoiceGenerated(current_view uint64, qc *flow.QuorumCertificate) { - t.pathHandler.NextStep(). - Uint64("block_view", current_view). - Msg("OnForkChoiceGenerated") - // Telemetry does not capture the details of the qc as the qc will be included in the - // proposed block, whose details (including the qc) are captured by telemetry +func (t *TelemetryConsumer) OnVoteProcessed(vote *model.Vote) { + t.noPathLogger.Info(). + Uint64("voted_block_view", vote.View). + Hex("voted_block_id", logging.ID(vote.BlockID)). + Hex("signer_id", logging.ID(vote.SignerID)). + Msg("OnVoteProcessed") } -func (t *TelemetryConsumer) OnQcConstructedFromVotes(curView uint64, qc *flow.QuorumCertificate) { - t.pathHandler.StartNextPath(curView) - t.pathHandler.NextStep(). - Uint64("curView", curView). - Uint64("qc_block_view", qc.View). - Hex("qc_block_id", qc.BlockID[:]). - Msg("OnQcConstructedFromVotes") +func (t *TelemetryConsumer) OnTimeoutProcessed(timeout *model.TimeoutObject) { + step := t.noPathLogger.Info(). + Uint64("view", timeout.View). + Uint64("timeout_tick", timeout.TimeoutTick). + Uint64("newest_qc_view", timeout.NewestQC.View). + Hex("newest_qc_block_id", logging.ID(timeout.NewestQC.BlockID)). + Hex("signer_id", logging.ID(timeout.SignerID)) + lastViewTC := timeout.LastViewTC + if lastViewTC != nil { + step. + Uint64("last_view_tc_view", lastViewTC.View). + Uint64("last_view_tc_newest_qc_view", lastViewTC.NewestQC.View). + Hex("last_view_tc_newest_qc_block_id", logging.ID(lastViewTC.NewestQC.BlockID)) + } + step.Msg("OnTimeoutProcessed") } -func (t *TelemetryConsumer) OnQcIncorporated(qc *flow.QuorumCertificate) { +func (t *TelemetryConsumer) OnCurrentViewDetails(currentView, finalizedView uint64, currentLeader flow.Identifier) { t.pathHandler.NextStep(). - Uint64("qc_block_view", qc.View). - Hex("qc_block_id", qc.BlockID[:]). - Msg("OnQcIncorporated") + Uint64("view", currentView). + Uint64("finalized_view", finalizedView). + Hex("current_leader", currentLeader[:]). + Msg("OnCurrentViewDetails") } // PathHandler maintains a notion of the current path through the state machine. @@ -180,8 +247,7 @@ func (t *TelemetryConsumer) OnQcIncorporated(qc *flow.QuorumCertificate) { // In case there is no currently open path, the PathHandler still returns a Step, // but such steps are logged as telemetry errors. type PathHandler struct { - chain flow.ChainID - log zerolog.Logger + log zerolog.Logger // currentPath holds a Zerolog Context with the information about the current path. // We represent the case where the current path has been closed by nil value. @@ -190,10 +256,10 @@ type PathHandler struct { // NewPathHandler instantiate a new PathHandler. // The PathHandler has no currently open path -func NewPathHandler(log zerolog.Logger, chain flow.ChainID) *PathHandler { +// Logger MUST include `chain` parameter as part of log context with corresponding chain ID to correctly map telemetry events to chain. +func NewPathHandler(log zerolog.Logger) *PathHandler { return &PathHandler{ - chain: chain, - log: log.With().Str("hotstuff", "telemetry").Str("chain", chain.String()).Logger(), + log: log.With().Str("component", "hotstuff.telemetry").Logger(), currentPath: nil, } } @@ -219,7 +285,7 @@ func (p *PathHandler) CloseCurrentPath() *PathHandler { return p } -// IsCurrentPathOpen if and only is the most recently started path has been closed. +// IsCurrentPathClosed if and only if the most recently started path has been closed. func (p *PathHandler) IsCurrentPathClosed() bool { return p.currentPath == nil } diff --git a/consensus/hotstuff/pacemaker.go b/consensus/hotstuff/pacemaker.go index caaaf7e06be..66b8787b241 100644 --- a/consensus/hotstuff/pacemaker.go +++ b/consensus/hotstuff/pacemaker.go @@ -1,12 +1,24 @@ package hotstuff import ( + "context" "time" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) +type LivenessData struct { + // CurrentView is the currently active view tracked by the PaceMaker. It is updated + // whenever the PaceMaker sees evidence (QC or TC) for advancing to next view. + CurrentView uint64 + // NewestQC is the newest QC (by view) observed by the PaceMaker. The QC can be observed on its own or as a part of TC. + NewestQC *flow.QuorumCertificate + // LastViewTC is the TC for the prior view (CurrentView-1), if this view timed out. If the previous round + // ended with a QC, this QC is stored in NewestQC and LastViewTC is nil. + LastViewTC *flow.TimeoutCertificate +} + // PaceMaker for HotStuff. The component is passive in that it only reacts to method calls. // The PaceMaker does not perform state transitions on its own. Timeouts are emitted through // channels. Each timeout has its own dedicated channel, which is garbage collected after the @@ -35,38 +47,42 @@ import ( // case <other events> // } // } +// +// Not concurrency safe. type PaceMaker interface { // CurView returns the current view. CurView() uint64 - // UpdateCurViewWithQC will check if the given QC will allow PaceMaker to fast - // forward to QC.view+1. If PaceMaker incremented the current View, a NewViewEvent will be returned. - UpdateCurViewWithQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, bool) + // NewestQC returns QC with the highest view discovered by PaceMaker. + NewestQC() *flow.QuorumCertificate - // UpdateCurViewWithBlock will check if the given block will allow PaceMaker to fast forward - // to the BlockProposal's view. If yes, the PaceMaker will update it's internal value for - // CurView and return a NewViewEvent. - // - // The parameter `nextPrimary` indicates to the PaceMaker whether or not this replica is the - // primary for the NEXT view taking block.view as reference. - // True corresponds to this replica being the next primary. - UpdateCurViewWithBlock(block *model.Block, isLeaderForNextView bool) (*model.NewViewEvent, bool) + // LastViewTC returns TC for last view, this could be nil if previous round + // has entered with a QC. + LastViewTC() *flow.TimeoutCertificate + + // ProcessQC will check if the given QC will allow PaceMaker to fast-forward to QC.view+1. + // If PaceMaker incremented the current View, a NewViewEvent will be returned. + // No errors are expected during normal operation. + ProcessQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, error) + + // ProcessTC will check if the given TC will allow PaceMaker to fast-forward to TC.view+1. + // If PaceMaker incremented the current View, a NewViewEvent will be returned. + // A nil TC is an expected valid input. + // No errors are expected during normal operation. + ProcessTC(tc *flow.TimeoutCertificate) (*model.NewViewEvent, error) // TimeoutChannel returns the timeout channel for the CURRENTLY ACTIVE timeout. - // Each time the pace maker starts a new timeout, this channel is replaced. + // Each time the pacemaker starts a new timeout, this channel is replaced. TimeoutChannel() <-chan time.Time - // OnTimeout is called when a timeout, which was previously created by the PaceMaker, has - // looped through the event loop. When used correctly, OnTimeout will always return - // a NewViewEvent. - // It is the responsibility of the calling code to ensure that NO STALE timeouts are - // delivered to the PaceMaker. - OnTimeout() *model.NewViewEvent - // Start starts the PaceMaker (i.e. the timeout for the configured starting value for view). - Start() + // CAUTION: EventHandler is not concurrency safe. The Start method must + // be executed by the same goroutine that also calls the other business logic + // methods, or concurrency safety has to be implemented externally. + Start(ctx context.Context) - // BlockRateDelay + // BlockRateDelay returns the minimal wait time for broadcasting a proposal, measured from + // the point in time when the primary (locally) enters the respective view. BlockRateDelay() time.Duration } diff --git a/consensus/hotstuff/pacemaker/pacemaker.go b/consensus/hotstuff/pacemaker/pacemaker.go index 72260a78756..8cb5ca3848e 100644 --- a/consensus/hotstuff/pacemaker/pacemaker.go +++ b/consensus/hotstuff/pacemaker/pacemaker.go @@ -1,157 +1,230 @@ package pacemaker import ( + "context" "fmt" "time" - "go.uber.org/atomic" - "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" "github.com/onflow/flow-go/model/flow" ) -// NitroPaceMaker implements the hotstuff.PaceMaker -// Its an aggressive pacemaker with exponential increase on timeout as well as -// exponential decrease on progress. Progress is defined as entering view V -// for which the replica knows a QC with V = QC.view + 1 -type NitroPaceMaker struct { - currentView uint64 +// ActivePaceMaker implements the hotstuff.PaceMaker +// Conceptually, we use the Pacemaker algorithm first proposed in [1] (specifically Jolteon) and described in more detail in [2]. +// [1] https://arxiv.org/abs/2106.10362 +// [2] https://developers.diem.com/papers/diem-consensus-state-machine-replication-in-the-diem-blockchain/2021-08-17.pdf (aka DiemBFT v4) +// +// To enter a new view `v`, the Pacemaker must observe a valid QC or TC for view `v-1`. +// The Pacemaker also controls when a node should locally time out for a given view. +// In contrast to the passive Pacemaker (previous implementation), locally timing a view +// does not cause a view change. +// A local timeout for a view `v` causes a node to: +// * never produce a vote for any proposal with view ≤ `v`, after the timeout +// * produce and broadcast a timeout object, which can form a part of the TC for the timed out view +// +// Not concurrency safe. +type ActivePaceMaker struct { + ctx context.Context timeoutControl *timeout.Controller notifier hotstuff.Consumer - started *atomic.Bool + persist hotstuff.Persister + livenessData *hotstuff.LivenessData + started bool } -// New creates a new NitroPaceMaker instance -// startView is the view for the pacemaker to start from -// timeoutController controls the timeout trigger. -// notifier provides callbacks for pacemaker events. -func New(startView uint64, timeoutController *timeout.Controller, notifier hotstuff.Consumer) (*NitroPaceMaker, error) { - if startView < 1 { - return nil, model.NewConfigurationErrorf("Please start PaceMaker with view > 0. (View 0 is reserved for genesis block, which has no proposer)") +var _ hotstuff.PaceMaker = (*ActivePaceMaker)(nil) + +// New creates a new ActivePaceMaker instance +// - startView is the view for the pacemaker to start with. +// - timeoutController controls the timeout trigger. +// - notifier provides callbacks for pacemaker events. +// +// Expected error conditions: +// * model.ConfigurationError if initial LivenessData is invalid +func New(timeoutController *timeout.Controller, + notifier hotstuff.Consumer, + persist hotstuff.Persister, +) (*ActivePaceMaker, error) { + livenessData, err := persist.GetLivenessData() + if err != nil { + return nil, fmt.Errorf("could not recover liveness data: %w", err) + } + + if livenessData.CurrentView < 1 { + return nil, model.NewConfigurationErrorf("PaceMaker cannot start in view 0 (view zero is reserved for genesis block, which has no proposer)") } - pm := NitroPaceMaker{ - currentView: startView, + pm := ActivePaceMaker{ + livenessData: livenessData, timeoutControl: timeoutController, notifier: notifier, - started: atomic.NewBool(false), + persist: persist, + started: false, } return &pm, nil } -// gotoView updates the current view to newView. Currently, the calling code +// updateLivenessData updates the current view, qc, tc. Currently, the calling code // ensures that the view number is STRICTLY monotonously increasing. The method -// gotoView panics as a last resort if FlowPaceMaker is modified to violate this condition. -// Hence, gotoView will _always_ return a NewViewEvent for an _increased_ view number. -func (p *NitroPaceMaker) gotoView(newView uint64) *model.NewViewEvent { - if newView <= p.currentView { +// updateLivenessData panics as a last resort if ActivePaceMaker is modified to violate this condition. +// No errors are expected, any error should be treated as exception. +func (p *ActivePaceMaker) updateLivenessData(newView uint64, qc *flow.QuorumCertificate, tc *flow.TimeoutCertificate) error { + if newView <= p.livenessData.CurrentView { // This should never happen: in the current implementation, it is trivially apparent that // newView is _always_ larger than currentView. This check is to protect the code from // future modifications that violate the necessary condition for // STRICTLY monotonously increasing view numbers. - panic(fmt.Sprintf("cannot move from view %d to %d: currentView must be strictly monotonously increasing", p.currentView, newView)) + return fmt.Errorf("cannot move from view %d to %d: currentView must be strictly monotonously increasing", + p.livenessData.CurrentView, newView) } - p.currentView = newView - timerInfo := p.timeoutControl.StartTimeout(model.ReplicaTimeout, newView) - p.notifier.OnStartingTimeout(timerInfo) - return &model.NewViewEvent{View: p.currentView} + + p.livenessData.CurrentView = newView + if p.livenessData.NewestQC.View < qc.View { + p.livenessData.NewestQC = qc + } + p.livenessData.LastViewTC = tc + err := p.persist.PutLivenessData(p.livenessData) + if err != nil { + return fmt.Errorf("could not persist liveness data: %w", err) + } + + return nil +} + +// updateNewestQC updates the highest QC tracked by view, iff `qc` has a larger view than +// the QC stored in the PaceMaker's `livenessData`. Otherwise, this method is a no-op. +// No errors are expected, any error should be treated as exception. +func (p *ActivePaceMaker) updateNewestQC(qc *flow.QuorumCertificate) error { + if p.livenessData.NewestQC.View >= qc.View { + return nil + } + + p.livenessData.NewestQC = qc + err := p.persist.PutLivenessData(p.livenessData) + if err != nil { + return fmt.Errorf("could not persist liveness data: %w", err) + } + + return nil } // CurView returns the current view -func (p *NitroPaceMaker) CurView() uint64 { - return p.currentView +func (p *ActivePaceMaker) CurView() uint64 { + return p.livenessData.CurrentView } // TimeoutChannel returns the timeout channel for current active timeout. // Note the returned timeout channel returns only one timeout, which is the current // timeout. // To get the timeout for the next timeout, you need to call TimeoutChannel() again. -func (p *NitroPaceMaker) TimeoutChannel() <-chan time.Time { +func (p *ActivePaceMaker) TimeoutChannel() <-chan time.Time { return p.timeoutControl.Channel() } -// UpdateCurViewWithQC notifies the pacemaker with a new QC, which might allow pacemaker to -// fast forward its view. -func (p *NitroPaceMaker) UpdateCurViewWithQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, bool) { - if qc.View < p.currentView { - return nil, false +// ProcessQC notifies the pacemaker with a new QC, which might allow pacemaker to +// fast-forward its view. In contrast to `ProcessTC`, this function does _not_ handle `nil` inputs. +// No errors are expected, any error should be treated as exception +func (p *ActivePaceMaker) ProcessQC(qc *flow.QuorumCertificate) (*model.NewViewEvent, error) { + oldView := p.CurView() + if qc.View < oldView { + err := p.updateNewestQC(qc) + if err != nil { + return nil, fmt.Errorf("could not update tracked newest QC: %w", err) + } + return nil, nil } - // qc.view = p.currentView + k for k ≥ 0 - // 2/3 of replicas have already voted for round p.currentView + k, hence proceeded past currentView - // => 2/3 of replicas are at least in view qc.view + 1. - // => replica can skip ahead to view qc.view + 1 + p.timeoutControl.OnProgressBeforeTimeout() + // supermajority of replicas have already voted during round `qc.view`, hence it is safe to proceed to subsequent view newView := qc.View + 1 - p.notifier.OnQcTriggeredViewChange(qc, newView) - return p.gotoView(newView), true + err := p.updateLivenessData(newView, qc, nil) + if err != nil { + return nil, err + } + + p.notifier.OnQcTriggeredViewChange(oldView, newView, qc) + p.notifier.OnViewChange(oldView, newView) + + timerInfo := p.timeoutControl.StartTimeout(p.ctx, newView) + p.notifier.OnStartingTimeout(timerInfo) + + return &model.NewViewEvent{ + View: timerInfo.View, + StartTime: timerInfo.StartTime, + Duration: timerInfo.Duration, + }, nil } -// UpdateCurViewWithBlock indicates the pacermaker that the block for the current view has received. -// and isLeaderForNextView indicates whether or not this replica is the primary for the NEXT view. -func (p *NitroPaceMaker) UpdateCurViewWithBlock(block *model.Block, isLeaderForNextView bool) (*model.NewViewEvent, bool) { - // use block's QC to fast-forward if possible - newViewOnQc, newViewOccurredOnQc := p.UpdateCurViewWithQC(block.QC) - if block.View != p.currentView { - return newViewOnQc, newViewOccurredOnQc +// ProcessTC notifies the Pacemaker of a new timeout certificate, which may allow +// Pacemaker to fast-forward its current view. +// A nil TC is an expected valid input, so that callers may pass in e.g. `Proposal.LastViewTC`, +// which may or may not have a value. +// No errors are expected, any error should be treated as exception +func (p *ActivePaceMaker) ProcessTC(tc *flow.TimeoutCertificate) (*model.NewViewEvent, error) { + if tc == nil { + return nil, nil } - // block is for current view - - if p.timeoutControl.TimerInfo().Mode != model.ReplicaTimeout { - // i.e. we are already on timeout.VoteCollectionTimeout. - // This edge case can occur as follows: - // * we previously already have processed a block for the current view - // and started the vote collection phase - // In this case, we do NOT want to RE-start the vote collection timer - // if we get a second block for the current View. - return nil, false - } - newViewOnBlock, newViewOccurredOnBlock := p.actOnBlockForCurView(block, isLeaderForNextView) - if !newViewOccurredOnBlock { // if processing current block didn't lead to NewView event, - // the initial processing of the block's QC still might have changes the view: - return newViewOnQc, newViewOccurredOnQc - } - // processing current block created NewView event, which is always newer than any potential newView event from processing the block's QC - return newViewOnBlock, newViewOccurredOnBlock -} -func (p *NitroPaceMaker) actOnBlockForCurView(block *model.Block, isLeaderForNextView bool) (*model.NewViewEvent, bool) { - if isLeaderForNextView { - timerInfo := p.timeoutControl.StartTimeout(model.VoteCollectionTimeout, p.currentView) - p.notifier.OnStartingTimeout(timerInfo) - return nil, false + oldView := p.CurView() + if tc.View < oldView { + err := p.updateNewestQC(tc.NewestQC) + if err != nil { + return nil, fmt.Errorf("could not update tracked newest QC: %w", err) + } + return nil, nil } - if block.QC.View+1 == p.currentView { - // only decrease timeout if block has been build on a quorum from the previous view; - // otherwise, the committee is still not synchronized (as the qc is from a view _prior_ to the previous one) - p.timeoutControl.OnProgressBeforeTimeout() + + p.timeoutControl.OnTimeout() + + // supermajority of replicas have already reached their timeout for view `tc.View`, hence it is safe to proceed to subsequent view + newView := tc.View + 1 + err := p.updateLivenessData(newView, tc.NewestQC, tc) + if err != nil { + return nil, err } - return p.gotoView(p.currentView + 1), true + + p.notifier.OnTcTriggeredViewChange(oldView, newView, tc) + p.notifier.OnViewChange(oldView, newView) + + timerInfo := p.timeoutControl.StartTimeout(p.ctx, newView) + p.notifier.OnStartingTimeout(timerInfo) + + return &model.NewViewEvent{ + View: timerInfo.View, + StartTime: timerInfo.StartTime, + Duration: timerInfo.Duration, + }, nil } -// OnTimeout notifies the pacemaker that the timeout event has looped through the event loop. -// It always trigger a view change, and the new view will be returned as NewViewEvent -func (p *NitroPaceMaker) OnTimeout() *model.NewViewEvent { - p.emitTimeoutNotifications(p.timeoutControl.TimerInfo()) - p.timeoutControl.OnTimeout() - return p.gotoView(p.currentView + 1) +// NewestQC returns QC with the highest view discovered by PaceMaker. +func (p *ActivePaceMaker) NewestQC() *flow.QuorumCertificate { + return p.livenessData.NewestQC } -func (p *NitroPaceMaker) emitTimeoutNotifications(timeout *model.TimerInfo) { - p.notifier.OnReachedTimeout(timeout) +// LastViewTC returns TC for last view, this will be nil only if the current view +// was entered with a QC. +func (p *ActivePaceMaker) LastViewTC() *flow.TimeoutCertificate { + return p.livenessData.LastViewTC } -// Start starts the pacemaker -func (p *NitroPaceMaker) Start() { - if p.started.Swap(true) { +// Start starts the pacemaker by starting the initial timer for the current view. +// Start should only be called once - subsequent calls are a no-op. +// CAUTION: ActivePaceMaker is not concurrency safe. The Start method must +// be executed by the same goroutine that also calls the other business logic +// methods, or concurrency safety has to be implemented externally. +func (p *ActivePaceMaker) Start(ctx context.Context) { + if p.started { return } - timerInfo := p.timeoutControl.StartTimeout(model.ReplicaTimeout, p.currentView) + p.started = true + p.ctx = ctx + timerInfo := p.timeoutControl.StartTimeout(ctx, p.CurView()) p.notifier.OnStartingTimeout(timerInfo) } // BlockRateDelay returns the delay for broadcasting its own proposals. -func (p *NitroPaceMaker) BlockRateDelay() time.Duration { +func (p *ActivePaceMaker) BlockRateDelay() time.Duration { return p.timeoutControl.BlockRateDelay() } diff --git a/consensus/hotstuff/pacemaker/pacemaker_test.go b/consensus/hotstuff/pacemaker/pacemaker_test.go index e570b51afe1..b0a8f70861d 100644 --- a/consensus/hotstuff/pacemaker/pacemaker_test.go +++ b/consensus/hotstuff/pacemaker/pacemaker_test.go @@ -1,16 +1,17 @@ package pacemaker import ( - "fmt" - "math" + "context" + "errors" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" @@ -18,475 +19,277 @@ import ( ) const ( - startRepTimeout float64 = 400.0 // Milliseconds - minRepTimeout float64 = 100.0 // Milliseconds - voteTimeoutFraction float64 = 0.5 // multiplicative factor - multiplicativeIncrease float64 = 1.5 // multiplicative factor - multiplicativeDecrease float64 = 0.85 // multiplicative factor + minRepTimeout float64 = 100.0 // Milliseconds + maxRepTimeout float64 = 600.0 // Milliseconds + multiplicativeIncrease float64 = 1.5 // multiplicative factor + happyPathMaxRoundFailures uint64 = 6 // number of failed rounds before first timeout increase ) -func expectedTimerInfo(view uint64, mode model.TimeoutMode) interface{} { +func expectedTimerInfo(view uint64) interface{} { return mock.MatchedBy( - func(timerInfo *model.TimerInfo) bool { - return timerInfo.View == view && timerInfo.Mode == mode + func(timerInfo model.TimerInfo) bool { + return timerInfo.View == view }) } -func expectedTimeoutInfo(view uint64, mode model.TimeoutMode) interface{} { - return mock.MatchedBy( - func(timerInfo *model.TimerInfo) bool { - return timerInfo.View == view && timerInfo.Mode == mode - }) +func TestActivePaceMaker(t *testing.T) { + suite.Run(t, new(ActivePaceMakerTestSuite)) +} + +type ActivePaceMakerTestSuite struct { + suite.Suite + + livenessData *hotstuff.LivenessData + notifier *mocks.Consumer + persist *mocks.Persister + paceMaker *ActivePaceMaker + stop context.CancelFunc } -func initPaceMaker(t *testing.T, view uint64) (hotstuff.PaceMaker, *mocks.Consumer) { - notifier := &mocks.Consumer{} +func (s *ActivePaceMakerTestSuite) SetupTest() { + s.notifier = mocks.NewConsumer(s.T()) + s.persist = mocks.NewPersister(s.T()) + tc, err := timeout.NewConfig( - time.Duration(startRepTimeout*1e6), time.Duration(minRepTimeout*1e6), - voteTimeoutFraction, + time.Duration(maxRepTimeout*1e6), multiplicativeIncrease, - multiplicativeDecrease, - 0) - if err != nil { - t.Fail() + happyPathMaxRoundFailures, + 0, + time.Duration(maxRepTimeout*1e6)) + require.NoError(s.T(), err) + + s.livenessData = &hotstuff.LivenessData{ + CurrentView: 3, + LastViewTC: nil, + NewestQC: helper.MakeQC(helper.WithQCView(2)), } - pm, err := New(view, timeout.NewController(tc), notifier) - if err != nil { - t.Fail() - } - notifier.On("OnStartingTimeout", expectedTimerInfo(view, model.ReplicaTimeout)).Return().Once() - pm.Start() - return pm, notifier -} -func QC(view uint64) *flow.QuorumCertificate { - return &flow.QuorumCertificate{View: view} -} + s.persist.On("GetLivenessData").Return(s.livenessData, nil).Once() -func makeBlock(qcView, blockView uint64) *model.Block { - return &model.Block{View: blockView, QC: QC(qcView)} -} + s.paceMaker, err = New(timeout.NewController(tc), s.notifier, s.persist) + require.NoError(s.T(), err) -// Test_SkipIncreaseViewThroughQC tests that PaceMaker increases View when receiving QC, -// if applicable, by skipping views -func Test_SkipIncreaseViewThroughQC(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - - qc := QC(3) - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", qc, uint64(4)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithQC(qc) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 4) - - qc = QC(12) - notifier.On("OnStartingTimeout", expectedTimerInfo(13, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", qc, uint64(13)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithQC(qc) - assert.True(t, nveOccurred && nve.View == 13) - - notifier.AssertExpectations(t) - assert.Equal(t, uint64(13), pm.CurView()) -} + s.notifier.On("OnStartingTimeout", expectedTimerInfo(s.livenessData.CurrentView)).Return().Once() -// Test_IgnoreOldBlocks tests that PaceMaker ignores old blocks -func Test_IgnoreOldQC(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - nve, nveOccurred := pm.UpdateCurViewWithQC(QC(2)) - assert.True(t, !nveOccurred && nve == nil) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(3), pm.CurView()) + var ctx context.Context + ctx, s.stop = context.WithCancel(context.Background()) + s.paceMaker.Start(ctx) } -// Test_SkipViewThroughBlock tests that PaceMaker skips View when receiving Block containing QC with larger View Number -func Test_SkipViewThroughBlock(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - - block := makeBlock(5, 9) - notifier.On("OnStartingTimeout", expectedTimerInfo(6, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(6)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(6), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 6) - - block = makeBlock(22, 25) - notifier.On("OnStartingTimeout", expectedTimerInfo(23, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(23)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, false) - assert.True(t, nveOccurred && nve.View == 23) - - notifier.AssertExpectations(t) - assert.Equal(t, uint64(23), pm.CurView()) +func (s *ActivePaceMakerTestSuite) TearDownTest() { + s.stop() } -// Test_HandlesSkipViewAttack verifies that PaceMaker skips views based on QC.view -// but NOT based on block.View to avoid vulnerability against Fast-Forward Attack -func Test_HandlesSkipViewAttack(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - - block := makeBlock(5, 9) - notifier.On("OnStartingTimeout", expectedTimerInfo(6, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(6)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(6), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 6) - - block = makeBlock(14, 23) - notifier.On("OnStartingTimeout", expectedTimerInfo(15, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(15)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, false) - assert.True(t, nveOccurred && nve.View == 15) - - notifier.AssertExpectations(t) - assert.Equal(t, uint64(15), pm.CurView()) +func QC(view uint64) *flow.QuorumCertificate { + return &flow.QuorumCertificate{View: view} } -// Test_IgnoreOldBlocks tests that PaceMaker ignores old blocks -func Test_IgnoreOldBlocks(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - pm.UpdateCurViewWithBlock(makeBlock(1, 2), false) - pm.UpdateCurViewWithBlock(makeBlock(1, 2), true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(3), pm.CurView()) +func LivenessData(qc *flow.QuorumCertificate) *hotstuff.LivenessData { + return &hotstuff.LivenessData{ + CurrentView: qc.View + 1, + LastViewTC: nil, + NewestQC: qc, + } } -// Test_ProcessBlockForCurrentView tests that PaceMaker processes the block for the current view correctly -func Test_ProcessBlockForCurrentView(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) - notifier.On("OnStartingTimeout", expectedTimerInfo(3, model.VoteCollectionTimeout)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(makeBlock(1, 3), true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(3), pm.CurView()) - assert.True(t, !nveOccurred && nve == nil) - - pm, notifier = initPaceMaker(t, 3) - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(makeBlock(1, 3), false) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 4) +// TestProcessQC_SkipIncreaseViewThroughQC tests that ActivePaceMaker increases view when receiving QC, +// if applicable, by skipping views +func (s *ActivePaceMakerTestSuite) TestProcessQC_SkipIncreaseViewThroughQC() { + qc := QC(s.livenessData.CurrentView) + s.persist.On("PutLivenessData", LivenessData(qc)).Return(nil).Once() + s.notifier.On("OnStartingTimeout", expectedTimerInfo(4)).Return().Once() + s.notifier.On("OnQcTriggeredViewChange", s.livenessData.CurrentView, uint64(4), qc).Return().Once() + s.notifier.On("OnViewChange", s.livenessData.CurrentView, qc.View+1).Once() + nve, err := s.paceMaker.ProcessQC(qc) + require.NoError(s.T(), err) + require.Equal(s.T(), qc.View+1, s.paceMaker.CurView()) + require.True(s.T(), nve.View == qc.View+1) + require.Equal(s.T(), qc, s.paceMaker.NewestQC()) + require.Nil(s.T(), s.paceMaker.LastViewTC()) + + // skip 10 views + qc = QC(s.livenessData.CurrentView + 10) + s.persist.On("PutLivenessData", LivenessData(qc)).Return(nil).Once() + s.notifier.On("OnStartingTimeout", expectedTimerInfo(qc.View+1)).Return().Once() + s.notifier.On("OnQcTriggeredViewChange", s.livenessData.CurrentView, qc.View+1, qc).Return().Once() + s.notifier.On("OnViewChange", s.livenessData.CurrentView, qc.View+1).Once() + nve, err = s.paceMaker.ProcessQC(qc) + require.NoError(s.T(), err) + require.True(s.T(), nve.View == qc.View+1) + require.Equal(s.T(), qc, s.paceMaker.NewestQC()) + require.Nil(s.T(), s.paceMaker.LastViewTC()) + + require.Equal(s.T(), qc.View+1, s.paceMaker.CurView()) } -// Test_FutureBlockWithQcForCurrentView tests that PaceMaker processes the block with -// -// block.qc.view = curView -// block.view = curView +1 -// -// correctly. Specifically, it should induce a view change to the block.view, which -// enables processing the block right away, i.e. switch to block.view + 1 -func Test_FutureBlockWithQcForCurrentView(t *testing.T) { - block := makeBlock(3, 4) - - // NOT Primary for the Block's view - pm, notifier := initPaceMaker(t, 3) - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(5, model.ReplicaTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(4)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, false) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(5), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 5) - - // PRIMARY for the Block's view - pm, notifier = initPaceMaker(t, 3) - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.VoteCollectionTimeout)).Return().Once() - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(4)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 4) +// TestProcessTC_SkipIncreaseViewThroughTC tests that ActivePaceMaker increases view when receiving TC, +// if applicable, by skipping views +func (s *ActivePaceMakerTestSuite) TestProcessTC_SkipIncreaseViewThroughTC() { + tc := helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView), + helper.WithTCNewestQC(s.livenessData.NewestQC)) + expectedLivenessData := &hotstuff.LivenessData{ + CurrentView: tc.View + 1, + LastViewTC: tc, + NewestQC: tc.NewestQC, + } + s.persist.On("PutLivenessData", expectedLivenessData).Return(nil).Once() + s.notifier.On("OnStartingTimeout", expectedTimerInfo(tc.View+1)).Return().Once() + s.notifier.On("OnTcTriggeredViewChange", s.livenessData.CurrentView, tc.View+1, tc).Return().Once() + s.notifier.On("OnViewChange", s.livenessData.CurrentView, tc.View+1).Once() + nve, err := s.paceMaker.ProcessTC(tc) + require.NoError(s.T(), err) + require.Equal(s.T(), tc.View+1, s.paceMaker.CurView()) + require.True(s.T(), nve.View == tc.View+1) + require.Equal(s.T(), tc, s.paceMaker.LastViewTC()) + + // skip 10 views + tc = helper.MakeTC(helper.WithTCView(tc.View+10), + helper.WithTCNewestQC(s.livenessData.NewestQC), + helper.WithTCNewestQC(QC(s.livenessData.CurrentView))) + expectedLivenessData = &hotstuff.LivenessData{ + CurrentView: tc.View + 1, + LastViewTC: tc, + NewestQC: tc.NewestQC, + } + s.persist.On("PutLivenessData", expectedLivenessData).Return(nil).Once() + s.notifier.On("OnStartingTimeout", expectedTimerInfo(tc.View+1)).Return().Once() + s.notifier.On("OnTcTriggeredViewChange", s.livenessData.CurrentView, tc.View+1, tc).Return().Once() + s.notifier.On("OnViewChange", s.livenessData.CurrentView, tc.View+1).Once() + nve, err = s.paceMaker.ProcessTC(tc) + require.NoError(s.T(), err) + require.True(s.T(), nve.View == tc.View+1) + require.Equal(s.T(), tc, s.paceMaker.LastViewTC()) + require.Equal(s.T(), tc.NewestQC, s.paceMaker.NewestQC()) + + require.Equal(s.T(), tc.View+1, s.paceMaker.CurView()) } -// Test_FutureBlockWithQcForCurrentView tests that PaceMaker processes the block with -// -// block.qc.view > curView -// block.view = block.qc.view +1 -// -// correctly. Specifically, it should induce a view change to block.view, which -// enables processing the block right away, i.e. switch to block.view + 1 -func Test_FutureBlockWithQcForFutureView(t *testing.T) { - block := makeBlock(13, 14) - - pm, notifier := initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(14)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(14, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(15, model.ReplicaTimeout)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, false) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(15), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 15) - - pm, notifier = initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(14)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(14, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(14, model.VoteCollectionTimeout)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(14), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 14) +// TestProcessTC_IgnoreOldTC tests that ActivePaceMaker ignores old TC and doesn't advance round. +func (s *ActivePaceMakerTestSuite) TestProcessTC_IgnoreOldTC() { + nve, err := s.paceMaker.ProcessTC(helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView-1), + helper.WithTCNewestQC(s.livenessData.NewestQC))) + require.NoError(s.T(), err) + require.Nil(s.T(), nve) + require.Equal(s.T(), s.livenessData.CurrentView, s.paceMaker.CurView()) } -// Test_FutureBlockWithQcForCurrentView tests that PaceMaker processes the block with -// -// block.qc.view > curView -// block.view > block.qc.view +1 -// -// correctly. Specifically, it should induce a view change to the block.qc.view +1, -// which is not sufficient to process the block. I.e. we expect view change to block.qc.view +1 -// enables processing the block right away. -func Test_FutureBlockWithQcForFutureFutureView(t *testing.T) { - block := makeBlock(13, 17) - - pm, notifier := initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(14)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(14, model.ReplicaTimeout)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, false) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(14), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 14) - - pm, notifier = initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(14)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(14, model.ReplicaTimeout)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(14), pm.CurView()) - assert.True(t, nveOccurred && nve.View == 14) +// TestProcessTC_IgnoreNilTC tests that ActivePaceMaker accepts nil TC as allowed input but doesn't trigger a new view event +func (s *ActivePaceMakerTestSuite) TestProcessTC_IgnoreNilTC() { + nve, err := s.paceMaker.ProcessTC(nil) + require.NoError(s.T(), err) + require.Nil(s.T(), nve) + require.Equal(s.T(), s.livenessData.CurrentView, s.paceMaker.CurView()) } -// Test_IgnoreBlockDuplicates tests that PaceMaker ignores duplicate blocks -func Test_IgnoreBlockDuplicates(t *testing.T) { - block := makeBlock(3, 4) - - // NOT Primary for the Block's view - pm, notifier := initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(4)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(5, model.ReplicaTimeout)).Return().Once() - nve, nveOccurred := pm.UpdateCurViewWithBlock(block, false) - assert.True(t, nveOccurred && nve.View == 5) - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, false) - assert.True(t, !nveOccurred && nve == nil) - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, false) - assert.True(t, !nveOccurred && nve == nil) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(5), pm.CurView()) - - // PRIMARY for the Block's view - pm, notifier = initPaceMaker(t, 3) - notifier.On("OnQcTriggeredViewChange", block.QC, uint64(4)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.VoteCollectionTimeout)).Return().Once() - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - assert.True(t, nveOccurred && nve.View == 4) - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - assert.True(t, !nveOccurred && nve == nil) - nve, nveOccurred = pm.UpdateCurViewWithBlock(block, true) - assert.True(t, !nveOccurred && nve == nil) - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) +// TestProcessQC_PersistException tests that ActivePaceMaker propagates exception +// when processing QC +func (s *ActivePaceMakerTestSuite) TestProcessQC_PersistException() { + exception := errors.New("persist-exception") + qc := QC(s.livenessData.CurrentView) + s.persist.On("PutLivenessData", mock.Anything).Return(exception).Once() + nve, err := s.paceMaker.ProcessQC(qc) + require.Nil(s.T(), nve) + require.ErrorIs(s.T(), err, exception) } -// Test_ReplicaTimeout tests that replica timeout fires as expected -func Test_ReplicaTimeout(t *testing.T) { - start := time.Now() - pm, notifier := initPaceMaker(t, 3) // initPaceMaker also calls Start() on PaceMaker - - select { - case <-pm.TimeoutChannel(): - break // testing path: corresponds to EventLoop picking up timeout from channel - case <-time.After(time.Duration(2) * time.Duration(startRepTimeout) * time.Millisecond): - t.Fail() // to prevent test from hanging - } - actualTimeout := float64(time.Since(start).Milliseconds()) // in millisecond - fmt.Println(actualTimeout) - assert.GreaterOrEqual(t, actualTimeout, startRepTimeout, "the actual timeout should be greater or equal to the init timeout") - // While the timeout event has been put in the channel, - // PaceMaker should NOT react on it without the timeout event being processed by the EventHandler - assert.Equal(t, uint64(3), pm.CurView()) - - // here the, the Event loop would now call EventHandler.OnTimeout() -> PaceMaker.OnTimeout() - notifier.On("OnReachedTimeout", expectedTimeoutInfo(3, model.ReplicaTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - viewChange := pm.OnTimeout() - if viewChange == nil { - assert.Fail(t, "Expecting ViewChange event as result of timeout") - } - - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) +// TestProcessTC_PersistException tests that ActivePaceMaker propagates exception +// when processing TC +func (s *ActivePaceMakerTestSuite) TestProcessTC_PersistException() { + exception := errors.New("persist-exception") + tc := helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView)) + s.persist.On("PutLivenessData", mock.Anything).Return(exception).Once() + nve, err := s.paceMaker.ProcessTC(tc) + require.Nil(s.T(), nve) + require.ErrorIs(s.T(), err, exception) } -// Test_ViewChangeWithProgress tests that the PaceMaker respects the definition of Progress: -// Progress is defined as entering view V for which the replica knows a QC with V = QC.view + 1 -// For this test, we feed it with a block for the current view containing a QC from the last view. -// Hence, the PaceMaker should decrease the timeout (because the committee is synchronized). -func Test_ViewChangeWithProgress(t *testing.T) { - pm, notifier := initPaceMaker(t, 5) // initPaceMaker also calls Start() on PaceMaker - - // The pace maker should transition into view 6 - // and decrease its timeout value by the following pacemaker call, as progress is made - notifier.On("OnStartingTimeout", expectedTimerInfo(6, model.ReplicaTimeout)).Return().Once() - start := time.Now() - nve, nveOccurred := pm.UpdateCurViewWithBlock(makeBlock(4, 5), false) - assert.True(t, nveOccurred && nve.View == 6) - notifier.AssertExpectations(t) - - select { - case <-pm.TimeoutChannel(): - break // testing path: corresponds to EventLoop picking up timeout from channel - case <-time.After(time.Duration(2) * time.Duration(startRepTimeout) * time.Millisecond): - t.Fail() // to prevent test from hanging - } - - actualTimeout := float64(time.Since(start).Milliseconds()) // in millisecond - expectedVoteCollectionTimeout := startRepTimeout * multiplicativeDecrease - assert.True(t, math.Abs(actualTimeout-expectedVoteCollectionTimeout) < 0.1*expectedVoteCollectionTimeout) - assert.Equal(t, uint64(6), pm.CurView()) +// TestProcessQC_InvalidatesLastViewTC verifies that PaceMaker does not retain any old +// TC if the last view change was triggered by observing a QC from the previous view. +func (s *ActivePaceMakerTestSuite) TestProcessQC_InvalidatesLastViewTC() { + tc := helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView+1), + helper.WithTCNewestQC(s.livenessData.NewestQC)) + s.persist.On("PutLivenessData", mock.Anything).Return(nil).Times(2) + s.notifier.On("OnStartingTimeout", mock.Anything).Return().Times(2) + s.notifier.On("OnTcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return().Once() + s.notifier.On("OnQcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return().Once() + s.notifier.On("OnViewChange", s.livenessData.CurrentView, tc.View+1).Once() + nve, err := s.paceMaker.ProcessTC(tc) + require.NotNil(s.T(), nve) + require.NoError(s.T(), err) + require.NotNil(s.T(), s.paceMaker.LastViewTC()) + + qc := QC(tc.View + 1) + s.notifier.On("OnViewChange", s.livenessData.CurrentView, qc.View+1).Once() + nve, err = s.paceMaker.ProcessQC(qc) + require.NotNil(s.T(), nve) + require.NoError(s.T(), err) + require.Nil(s.T(), s.paceMaker.LastViewTC()) } -// Test_ViewChangeWithoutProgress tests that the PaceMaker respects the definition of Progress: -// Progress is defined as entering view V for which the replica knows a QC with V = QC.view + 1 -// For this test, we feed it with a block for the current view, but containing an old QC. -// Hence, the PaceMaker should NOT decrease the timeout (because the committee is still not synchronized) -func Test_ViewChangeWithoutProgress(t *testing.T) { - pm, notifier := initPaceMaker(t, 5) // initPaceMaker also calls Start() on PaceMaker - - // while the pace maker should transition into view 6 - // we are not expecting a change of timeout value by the following pacemaker call, because no progress is made - notifier.On("OnStartingTimeout", expectedTimerInfo(6, model.ReplicaTimeout)).Return().Once() - start := time.Now() - nve, nveOccurred := pm.UpdateCurViewWithBlock(makeBlock(3, 5), false) - assert.True(t, nveOccurred && nve.View == 6) - notifier.AssertExpectations(t) - - select { - case <-pm.TimeoutChannel(): - break // testing path: corresponds to EventLoop picking up timeout from channel - case <-time.After(time.Duration(2) * time.Duration(startRepTimeout) * time.Millisecond): - t.Fail() // to prevent test from hanging - } - - actualTimeout := float64(time.Since(start).Milliseconds()) // in millisecond - expectedVoteCollectionTimeout := startRepTimeout - assert.True(t, math.Abs(actualTimeout-expectedVoteCollectionTimeout) < 0.1*expectedVoteCollectionTimeout) - assert.Equal(t, uint64(6), pm.CurView()) +// TestProcessQC_IgnoreOldQC tests that ActivePaceMaker ignores old QC and doesn't advance round +func (s *ActivePaceMakerTestSuite) TestProcessQC_IgnoreOldQC() { + qc := QC(s.livenessData.CurrentView - 1) + nve, err := s.paceMaker.ProcessQC(qc) + require.NoError(s.T(), err) + require.Nil(s.T(), nve) + require.Equal(s.T(), s.livenessData.CurrentView, s.paceMaker.CurView()) + require.NotEqual(s.T(), qc, s.paceMaker.NewestQC()) } -func Test_ReplicaTimeoutAgain(t *testing.T) { - start := time.Now() - pm, notifier := initPaceMaker(t, 3) // initPaceMaker also calls Start() on PaceMaker - notifier.On("OnReachedTimeout", mock.Anything) - notifier.On("OnStartingTimeout", mock.Anything) - - // wait until the timeout is hit for the first time - select { - case <-pm.TimeoutChannel(): - case <-time.After(time.Duration(2*startRepTimeout) * time.Millisecond): - } - - // calculate the actual timeout duration that has been waited. - actualTimeout := float64(time.Since(start).Milliseconds()) // in millisecond - assert.GreaterOrEqual(t, actualTimeout, startRepTimeout, "the actual timeout should be greater or equal to the init timeout") - assert.Less(t, actualTimeout, 2*startRepTimeout, "the actual timeout was too long") - - // reset timer - start = time.Now() - - // trigger view change and restart the pacemaker timer. The next timeout should take 1.5 longer - _ = pm.OnTimeout() - - // wait until the timeout is hit again - select { - case <-pm.TimeoutChannel(): - case <-time.After(time.Duration(3) * time.Duration(startRepTimeout) * time.Millisecond): - } - - nv := pm.OnTimeout() - - // calculate the actual timeout duration that has been waited again - actualTimeout = float64(time.Since(start).Milliseconds()) // in millisecond - assert.GreaterOrEqual(t, actualTimeout, 1.5*startRepTimeout, "the actual timeout should be greater or equal to the timeout") - assert.Less(t, actualTimeout, 2*startRepTimeout, "the actual timeout was too long") - - var changed bool - // timeout reduce linearly - select { - case <-pm.TimeoutChannel(): - break - case <-time.After(time.Duration(0.1 * startRepTimeout)): - nv, changed = pm.UpdateCurViewWithBlock(makeBlock(nv.View-1, nv.View), false) - break - } - - require.True(t, changed) - - // timeout reduce linearly - select { - case <-pm.TimeoutChannel(): - break - case <-time.After(time.Duration(0.1 * startRepTimeout)): - // start the timer before the UpdateCurViewWithBlock is called, which starts the internal timer. - // This is required to make the tests more accurate, because when it's running on CI, - // there might be some delay for time.Now() gets called, which impact the accuracy of the actual timeout - // measurement - - start = time.Now() - _, changed = pm.UpdateCurViewWithBlock(makeBlock(nv.View-1, nv.View), false) - break +// TestProcessQC_UpdateNewestQC tests that ActivePaceMaker tracks the newest QC even if it has advanced past this view. +// In this test, we feed a newer QC as part of a TC into the PaceMaker. +func (s *ActivePaceMakerTestSuite) TestProcessQC_UpdateNewestQC() { + s.persist.On("PutLivenessData", mock.Anything).Return(nil).Once() + s.notifier.On("OnStartingTimeout", mock.Anything).Return().Once() + s.notifier.On("OnTcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return().Once() + tc := helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView+10), + helper.WithTCNewestQC(s.livenessData.NewestQC)) + s.notifier.On("OnViewChange", s.livenessData.CurrentView, tc.View+1).Once() + nve, err := s.paceMaker.ProcessTC(tc) + require.NoError(s.T(), err) + require.NotNil(s.T(), nve) + + qc := QC(s.livenessData.NewestQC.View + 5) + + expectedLivenessData := &hotstuff.LivenessData{ + CurrentView: s.livenessData.CurrentView, + LastViewTC: s.livenessData.LastViewTC, + NewestQC: qc, } + s.persist.On("PutLivenessData", expectedLivenessData).Return(nil).Once() - require.True(t, changed) - - // wait until the timeout is hit again - select { - case <-pm.TimeoutChannel(): - case <-time.After(time.Duration(3) * time.Duration(startRepTimeout) * time.Millisecond): - } - - actualTimeout = float64(time.Since(start).Microseconds()) * 0.001 // in millisecond - // the actual timeout should be startRepTimeout * 1.5 *1.5 * multiplicativeDecrease * multiplicativeDecrease - // because it hits timeout twice and then received blocks for current view twice - assert.GreaterOrEqual(t, actualTimeout, 1.5*1.5*startRepTimeout*multiplicativeDecrease*multiplicativeDecrease, "the actual timeout should be greater or equal to the timeout") - assert.Less(t, actualTimeout, 1.5*1.5*startRepTimeout*multiplicativeDecrease, "the actual timeout is too long") + nve, err = s.paceMaker.ProcessQC(qc) + require.NoError(s.T(), err) + require.Nil(s.T(), nve) + require.Equal(s.T(), qc, s.paceMaker.NewestQC()) } -// Test_VoteTimeout tests that vote timeout fires as expected -func Test_VoteTimeout(t *testing.T) { - pm, notifier := initPaceMaker(t, 3) // initPaceMaker also calls Start() on PaceMaker - start := time.Now() - - notifier.On("OnStartingTimeout", expectedTimerInfo(3, model.VoteCollectionTimeout)).Return().Once() - // we are not expecting a change of timeout value by the following pacemaker call, because no view change happens - pm.UpdateCurViewWithBlock(makeBlock(2, 3), true) - notifier.AssertExpectations(t) - - // the pacemaker should now be running the vote collection timeout: - expectedVoteCollectionTimeout := startRepTimeout * voteTimeoutFraction - select { - case <-pm.TimeoutChannel(): - break // testing path: corresponds to EventLoop picking up timeout from channel - case <-time.After(2 * time.Duration(expectedVoteCollectionTimeout) * time.Millisecond): - t.Fail() // to prevent test from hanging +// TestProcessTC_UpdateNewestQC tests that ActivePaceMaker tracks the newest QC included in TC even if it has advanced past this view. +func (s *ActivePaceMakerTestSuite) TestProcessTC_UpdateNewestQC() { + s.persist.On("PutLivenessData", mock.Anything).Return(nil).Once() + s.notifier.On("OnStartingTimeout", mock.Anything).Return().Once() + s.notifier.On("OnTcTriggeredViewChange", mock.Anything, mock.Anything, mock.Anything).Return().Once() + tc := helper.MakeTC(helper.WithTCView(s.livenessData.CurrentView+10), + helper.WithTCNewestQC(s.livenessData.NewestQC)) + s.notifier.On("OnViewChange", s.livenessData.CurrentView, tc.View+1).Once() + nve, err := s.paceMaker.ProcessTC(tc) + require.NoError(s.T(), err) + require.NotNil(s.T(), nve) + + qc := QC(s.livenessData.NewestQC.View + 5) + olderTC := helper.MakeTC(helper.WithTCView(s.paceMaker.CurView()-1), + helper.WithTCNewestQC(qc)) + + expectedLivenessData := &hotstuff.LivenessData{ + CurrentView: s.livenessData.CurrentView, + LastViewTC: s.livenessData.LastViewTC, + NewestQC: qc, } - duration := float64(time.Since(start).Milliseconds()) // in millisecond - assert.True(t, math.Abs(duration-expectedVoteCollectionTimeout) < 0.1*expectedVoteCollectionTimeout) - // While the timeout event has been put in the channel, - // PaceMaker should NOT react on it without the timeout event being processed by the EventHandler - assert.Equal(t, uint64(3), pm.CurView()) - - // here the, the Event loop would now call EventHandler.OnTimeout() -> PaceMaker.OnTimeout() - notifier.On("OnReachedTimeout", expectedTimeoutInfo(3, model.VoteCollectionTimeout)).Return().Once() - notifier.On("OnStartingTimeout", expectedTimerInfo(4, model.ReplicaTimeout)).Return().Once() - viewChange := pm.OnTimeout() - if viewChange == nil { - assert.Fail(t, "Expecting ViewChange event as result of timeout") - } - notifier.AssertExpectations(t) - assert.Equal(t, uint64(4), pm.CurView()) + s.persist.On("PutLivenessData", expectedLivenessData).Return(nil).Once() + + nve, err = s.paceMaker.ProcessTC(olderTC) + require.NoError(s.T(), err) + require.Nil(s.T(), nve) + require.Equal(s.T(), qc, s.paceMaker.NewestQC()) } diff --git a/consensus/hotstuff/pacemaker/timeout/config.go b/consensus/hotstuff/pacemaker/timeout/config.go index 5df83571f79..6de384f92d3 100644 --- a/consensus/hotstuff/pacemaker/timeout/config.go +++ b/consensus/hotstuff/pacemaker/timeout/config.go @@ -2,7 +2,6 @@ package timeout import ( "fmt" - "math" "time" "github.com/rs/zerolog/log" @@ -12,26 +11,26 @@ import ( "github.com/onflow/flow-go/module/updatable_configs" ) -// Config contains the configuration parameters for ExponentialIncrease-LinearDecrease -// timeout.Controller -// - on timeout: increase timeout by multiplicative factor `timeoutIncrease` (user-specified) -// this results in exponential growing timeout duration on multiple subsequent timeouts -// - on progress: MULTIPLICATIVE timeout decrease +// Config contains the configuration parameters for a Truncated Exponential Backoff, +// as implemented by the `timeout.Controller` +// - On timeout: increase timeout by multiplicative factor `TimeoutAdjustmentFactor`. This +// results in exponentially growing timeout duration on multiple subsequent timeouts. +// - On progress: decrease timeout by multiplicative factor `TimeoutAdjustmentFactor. type Config struct { - // ReplicaTimeout is the duration of a view before we time out [MILLISECONDS] - // ReplicaTimeout is the only variable quantity - ReplicaTimeout float64 // MinReplicaTimeout is the minimum the timeout can decrease to [MILLISECONDS] MinReplicaTimeout float64 - // VoteAggregationTimeoutFraction is the FRACTION of ReplicaTimeout which the Primary - // will maximally wait to collect enough votes before building a block (with an old qc) - VoteAggregationTimeoutFraction float64 - // TimeoutDecrease: MULTIPLICATIVE factor for increasing timeout on timeout - TimeoutIncrease float64 - // TimeoutDecrease: MULTIPLICATIVE factor for decreasing timeout on progress - TimeoutDecrease float64 + // MaxReplicaTimeout is the maximum value the timeout can increase to [MILLISECONDS] + MaxReplicaTimeout float64 + // TimeoutAdjustmentFactor: MULTIPLICATIVE factor for increasing timeout when view + // change was triggered by a TC (unhappy path) or decreasing the timeout on progress + TimeoutAdjustmentFactor float64 + // HappyPathMaxRoundFailures is the number of rounds without progress where we still consider being + // on hot path of execution. After exceeding this value we will start increasing timeout values. + HappyPathMaxRoundFailures uint64 // BlockRateDelayMS is a delay to broadcast the proposal in order to control block production rate [MILLISECONDS] BlockRateDelayMS *atomic.Float64 + // MaxTimeoutObjectRebroadcastInterval is the maximum value for timeout object rebroadcast interval [MILLISECONDS] + MaxTimeoutObjectRebroadcastInterval float64 } var DefaultConfig = NewDefaultConfig() @@ -40,25 +39,25 @@ var DefaultConfig = NewDefaultConfig() // We explicitly provide a method here, which demonstrates in-code how // to compute standard values from some basic quantities. func NewDefaultConfig() Config { - // the replicas will start with 60 second time out to allow all other replicas to come online - // once the replica's views are synchronized, the timeout will decrease to more reasonable values - replicaTimeout := 60 * time.Second - - // the lower bound on the replicaTimeout value + // minReplicaTimeout is the lower bound on the replica's timeout value, this is also the initial timeout with what replicas + // will start their execution. // If HotStuff is running at full speed, 1200ms should be enough. However, we add some buffer. // This value is for instant message delivery. - minReplicaTimeout := 2 * time.Second - timeoutIncreaseFactor := 2.0 + minReplicaTimeout := 3 * time.Second + maxReplicaTimeout := 1 * time.Minute + timeoutAdjustmentFactorFactor := 1.2 + // after 6 successively failed rounds, the pacemaker leaves the hot path and starts increasing timeouts (recovery mode) + happyPathMaxRoundFailures := uint64(6) blockRateDelay := 0 * time.Millisecond + maxRebroadcastInterval := 5 * time.Second - // the following demonstrates the computation of standard values conf, err := NewConfig( - replicaTimeout, minReplicaTimeout+blockRateDelay, - StandardVoteAggregationTimeoutFraction(minReplicaTimeout, blockRateDelay), // resulting value here is 0.5 - timeoutIncreaseFactor, - StandardTimeoutDecreaseFactor(1.0/3.0, timeoutIncreaseFactor), // resulting value is 1/sqrt(2) + maxReplicaTimeout, + timeoutAdjustmentFactorFactor, + happyPathMaxRoundFailures, blockRateDelay, + maxRebroadcastInterval, ) if err != nil { // we check in a unit test that this does not happen @@ -69,47 +68,48 @@ func NewDefaultConfig() Config { } // NewConfig creates a new TimoutConfig. -// startReplicaTimeout: starting timeout value for replica round [Milliseconds]; -// minReplicaTimeout: minimal timeout value for replica round [Milliseconds]; -// voteAggregationTimeoutFraction: fraction of replicaTimeout which is reserved for aggregating votes; -// timeoutIncrease: multiplicative factor for increasing timeout; -// timeoutDecrease: linear subtrahend for timeout decrease [Milliseconds] -// blockRateDelay: a delay to delay the proposal broadcasting +// - minReplicaTimeout: minimal timeout value for replica round [Milliseconds] +// Consistency requirement: must be non-negative +// - maxReplicaTimeout: maximal timeout value for replica round [Milliseconds] +// Consistency requirement: must be non-negative and cannot be smaller than minReplicaTimeout +// - timeoutAdjustmentFactor: multiplicative factor for adjusting timeout duration +// Consistency requirement: must be strictly larger than 1 +// - happyPathMaxRoundFailures: number of successive failed rounds after which we will start increasing timeouts +// - blockRateDelay: a delay to delay the proposal broadcasting [Milliseconds] +// Consistency requirement: must be non-negative +// +// Returns `model.ConfigurationError` is any of the consistency requirements is violated. func NewConfig( - startReplicaTimeout time.Duration, minReplicaTimeout time.Duration, - voteAggregationTimeoutFraction float64, - timeoutIncrease float64, - timeoutDecrease float64, + maxReplicaTimeout time.Duration, + timeoutAdjustmentFactor float64, + happyPathMaxRoundFailures uint64, blockRateDelay time.Duration, + maxRebroadcastInterval time.Duration, ) (Config, error) { - if startReplicaTimeout < minReplicaTimeout { - return Config{}, model.NewConfigurationErrorf("startReplicaTimeout (%dms) cannot be smaller than minReplicaTimeout (%dms)", - startReplicaTimeout.Milliseconds(), minReplicaTimeout.Milliseconds()) - } - if minReplicaTimeout < 0 { - return Config{}, model.NewConfigurationErrorf("minReplicaTimeout must non-negative") + if minReplicaTimeout <= 0 { + return Config{}, model.NewConfigurationErrorf("minReplicaTimeout must be a positive number[milliseconds]") } - if voteAggregationTimeoutFraction <= 0 || 1 < voteAggregationTimeoutFraction { - return Config{}, model.NewConfigurationErrorf("VoteAggregationTimeoutFraction must be in range (0,1]") + if maxReplicaTimeout < minReplicaTimeout { + return Config{}, model.NewConfigurationErrorf("maxReplicaTimeout cannot be smaller than minReplicaTimeout") } - if timeoutIncrease <= 1 { - return Config{}, model.NewConfigurationErrorf("TimeoutIncrease must be strictly bigger than 1") - } - if timeoutDecrease <= 0 || 1 <= timeoutDecrease { - return Config{}, model.NewConfigurationErrorf("timeoutDecrease must be in range (0,1)") + if timeoutAdjustmentFactor <= 1 { + return Config{}, model.NewConfigurationErrorf("timeoutAdjustmentFactor must be strictly bigger than 1") } if err := validBlockRateDelay(blockRateDelay); err != nil { return Config{}, err } + if maxRebroadcastInterval <= 0 { + return Config{}, model.NewConfigurationErrorf("maxRebroadcastInterval must be a positive number [milliseconds]") + } tc := Config{ - ReplicaTimeout: float64(startReplicaTimeout.Milliseconds()), - MinReplicaTimeout: float64(minReplicaTimeout.Milliseconds()), - VoteAggregationTimeoutFraction: voteAggregationTimeoutFraction, - TimeoutIncrease: timeoutIncrease, - TimeoutDecrease: timeoutDecrease, - BlockRateDelayMS: atomic.NewFloat64(float64(blockRateDelay.Milliseconds())), + MinReplicaTimeout: float64(minReplicaTimeout.Milliseconds()), + MaxReplicaTimeout: float64(maxReplicaTimeout.Milliseconds()), + TimeoutAdjustmentFactor: timeoutAdjustmentFactor, + HappyPathMaxRoundFailures: happyPathMaxRoundFailures, + MaxTimeoutObjectRebroadcastInterval: float64(maxRebroadcastInterval.Milliseconds()), + BlockRateDelayMS: atomic.NewFloat64(float64(blockRateDelay.Milliseconds())), } return tc, nil } @@ -149,22 +149,3 @@ func (c *Config) SetBlockRateDelay(delay time.Duration) error { c.BlockRateDelayMS.Store(float64(delay.Milliseconds())) return nil } - -// StandardVoteAggregationTimeoutFraction calculates a standard value for the VoteAggregationTimeoutFraction in case a block delay is used. -// The motivation for the standard value is as follows: -// - the next primary receives the block it ideally would extend at some time t -// - the best guess the primary has, when other nodes would receive the block is at time t as well -// - the primary needs to get its block to the other replicas, before they time out: -// the primary uses its own timeout as estimator for the other replicas' timeout -func StandardVoteAggregationTimeoutFraction(minReplicaTimeout time.Duration, blockRateDelay time.Duration) float64 { - standardVoteAggregationTimeoutFraction := 0.5 - minReplicaTimeoutMS := float64(minReplicaTimeout.Milliseconds()) - blockRateDelayMS := float64(blockRateDelay.Milliseconds()) - return (standardVoteAggregationTimeoutFraction*minReplicaTimeoutMS + blockRateDelayMS) / (minReplicaTimeoutMS + blockRateDelayMS) -} - -// StandardTimeoutDecreaseFactor calculates a standard value for TimeoutDecreaseFactor -// for an assumed max fraction of offline (byzantine) HotStuff committee members -func StandardTimeoutDecreaseFactor(maxFractionOfflineReplicas, timeoutIncreaseFactor float64) float64 { - return math.Pow(timeoutIncreaseFactor, maxFractionOfflineReplicas/(maxFractionOfflineReplicas-1)) -} diff --git a/consensus/hotstuff/pacemaker/timeout/config_test.go b/consensus/hotstuff/pacemaker/timeout/config_test.go index 191d63703b7..259b87727ed 100644 --- a/consensus/hotstuff/pacemaker/timeout/config_test.go +++ b/consensus/hotstuff/pacemaker/timeout/config_test.go @@ -1,92 +1,56 @@ package timeout import ( - "math" "testing" "time" "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/model" ) +// TestConstructor tests that constructor performs needed checks and returns expected values depending on different inputs. func TestConstructor(t *testing.T) { - c, err := NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0.73, 1.5, 0.85, time.Second) + c, err := NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) require.NoError(t, err) - require.Equal(t, float64(2200), c.ReplicaTimeout) require.Equal(t, float64(1200), c.MinReplicaTimeout) - require.Equal(t, float64(0.73), c.VoteAggregationTimeoutFraction) - require.Equal(t, float64(1.5), c.TimeoutIncrease) - require.Equal(t, float64(0.85), c.TimeoutDecrease) + require.Equal(t, float64(2000), c.MaxReplicaTimeout) + require.Equal(t, float64(1.5), c.TimeoutAdjustmentFactor) + require.Equal(t, uint64(3), c.HappyPathMaxRoundFailures) require.Equal(t, float64(1000), c.BlockRateDelayMS.Load()) - - // should not allow startReplicaTimeout < minReplicaTimeout - _, err = NewConfig(800*time.Millisecond, 1200*time.Millisecond, 0.73, 1.5, 0.85, time.Second) - require.Error(t, err) + require.Equal(t, float64(2000), c.MaxTimeoutObjectRebroadcastInterval) // should not allow negative minReplicaTimeout - c, err = NewConfig(2200*time.Millisecond, -1200*time.Millisecond, 0.73, 1.5, 0.85, time.Second) - require.Error(t, err) + c, err = NewConfig(-1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) - // should not allow voteAggregationTimeoutFraction to be 0 or larger than 1 - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0, 1.5, 0.85, time.Second) - require.Error(t, err) - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 1.00001, 1.5, 0.85, time.Second) - require.Error(t, err) + // should not allow 0 minReplicaTimeout + c, err = NewConfig(0, 2000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) - // should not allow timeoutIncrease to be 1.0 or smaller - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0.73, 1.0, 0.85, time.Second) - require.Error(t, err) + // should not allow maxReplicaTimeout < minReplicaTimeout + c, err = NewConfig(1200*time.Millisecond, 1000*time.Millisecond, 1.5, 3, time.Second, 2000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) - // should not allow timeoutDecrease to be zero or 1.0 - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0.73, 1.5, 0, time.Second) - require.Error(t, err) - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0.73, 1.5, 1, time.Second) - require.Error(t, err) + // should not allow timeoutIncrease to be 1.0 or smaller + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.0, 3, time.Second, 2000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) // should not allow blockRateDelay to be zero negative - c, err = NewConfig(2200*time.Millisecond, 1200*time.Millisecond, 0.73, 1.5, 0.85, -1*time.Nanosecond) - require.Error(t, err) + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, -1*time.Nanosecond, 2000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) + + // should not allow maxRebroadcastInterval to be smaller than minReplicaTimeout + c, err = NewConfig(1200*time.Millisecond, 2000*time.Millisecond, 1.5, 3, -1*time.Nanosecond, 1000*time.Millisecond) + require.True(t, model.IsConfigurationError(err)) } +// TestDefaultConfig tests that default config is filled with correct values. func TestDefaultConfig(t *testing.T) { c := NewDefaultConfig() - require.Equal(t, float64(60000), c.ReplicaTimeout) - require.Equal(t, float64(2000), c.MinReplicaTimeout) - require.Equal(t, float64(0.5), c.VoteAggregationTimeoutFraction) - require.Equal(t, float64(2.0), c.TimeoutIncrease) - require.True(t, math.Abs(1/math.Sqrt(2)-c.TimeoutDecrease) < 1e-15) // need to allow for some numerical error + require.Equal(t, float64(3000), c.MinReplicaTimeout) + require.Equal(t, 1.2, c.TimeoutAdjustmentFactor) + require.Equal(t, uint64(6), c.HappyPathMaxRoundFailures) require.Equal(t, float64(0), c.BlockRateDelayMS.Load()) } - -// TestStandardVoteAggregationTimeoutFraction tests the computation of the standard -// value for `VoteAggregationTimeoutFraction`. -func TestStandardVoteAggregationTimeoutFraction(t *testing.T) { - // test numerical computation for one specific parameter setting - f := StandardVoteAggregationTimeoutFraction(1200*time.Millisecond, 500*time.Millisecond) - require.Equal(t, (0.5*1200.0+500.0)/(1200.0+500.0), f) - - // for no blockRateDelay, the standard value should be 0.5 - f = StandardVoteAggregationTimeoutFraction(123456*time.Millisecond, 0) - require.Equal(t, 0.5, f) - - // for _very_ large blockRateDelay, the standard value should converge to one - f = StandardVoteAggregationTimeoutFraction(123456*time.Millisecond, 10000*time.Hour) - require.True(t, f <= 1.0) - require.True(t, math.Abs(f-1.0) < 1e-5) -} - -func TestStandardTimeoutDecreaseFactor(t *testing.T) { - timeoutIncreaseFactor := 2.0 - - offlineFraction := 1.0 / 3.0 - f := StandardTimeoutDecreaseFactor(offlineFraction, timeoutIncreaseFactor) - expected := math.Pow(timeoutIncreaseFactor, offlineFraction) * math.Pow(f, 1.0-offlineFraction) - numericalError := math.Abs(expected - 1.0) - require.True(t, numericalError < 1e-15) - - offlineFraction = 0.2 - f = StandardTimeoutDecreaseFactor(offlineFraction, timeoutIncreaseFactor) - expected = math.Pow(timeoutIncreaseFactor, offlineFraction) * math.Pow(f, 1.0-offlineFraction) - numericalError = math.Abs(expected - 1.0) - require.True(t, numericalError < 1e-15) -} diff --git a/consensus/hotstuff/pacemaker/timeout/controller.go b/consensus/hotstuff/pacemaker/timeout/controller.go index 4d65f573d34..55c73137134 100644 --- a/consensus/hotstuff/pacemaker/timeout/controller.go +++ b/consensus/hotstuff/pacemaker/timeout/controller.go @@ -1,28 +1,43 @@ package timeout import ( + "context" "math" "time" "github.com/onflow/flow-go/consensus/hotstuff/model" ) -// Controller implements a timout with: -// - on timeout: increase timeout by multiplicative factor `timeoutIncrease` (user-specified) -// this results in exponential growing timeout duration on multiple subsequent timeouts -// - on progress: decrease timeout by subtrahend `timeoutDecrease` +// Controller implements the following truncated exponential backoff: +// +// duration = t_min * min(b ^ ((r-k) * θ(r-k)), t_max) +// +// For practical purpose we will transform this formula into: +// +// duration(r) = t_min * b ^ (min((r-k) * θ(r-k)), c), where c = log_b (t_max / t_min). +// +// In described formula: +// +// k - is number of rounds we expect during hot path, after failing this many rounds, +// we will start increasing timeouts. +// b - timeout increase factor +// r - failed rounds counter +// θ - Heaviside step function +// t_min/t_max - minimum/maximum round duration +// +// By manipulating `r` after observing progress or lack thereof, we are achieving exponential increase/decrease +// of round durations. +// - on timeout: increase number of failed rounds, this results in exponential growing round duration +// on multiple subsequent timeouts, after exceeding k. +// - on progress: decrease number of failed rounds, this results in exponential decrease of round duration. type Controller struct { cfg Config - timer *time.Timer - timerInfo *model.TimerInfo - timeoutChannel <-chan time.Time + timeoutChannel chan time.Time + stopTicker context.CancelFunc + maxExponent float64 // max exponent for exponential function, derived from maximum round duration + r uint64 // failed rounds counter, higher value results in longer round duration } -// timeoutCap this is an internal cap on the timeout to avoid numerical overflows. -// Its value is large enough to be of no practical implication. -// We use 1E9 milliseconds which is about 11 days for a single timout (i.e. more than a full epoch) -const timeoutCap float64 = 1e9 - // NewController creates a new Controller. func NewController(timeoutConfig Config) *Controller { // the initial value for the timeout channel is a closed channel which returns immediately @@ -30,81 +45,105 @@ func NewController(timeoutConfig Config) *Controller { startChannel := make(chan time.Time) close(startChannel) + // we need to calculate log_b(t_max/t_min), golang doesn't support logarithm with custom base + // we will apply change of base logarithm transformation to get around this: + // log_b(x) = log_e(x) / log_e(b) + maxExponent := math.Log(timeoutConfig.MaxReplicaTimeout/timeoutConfig.MinReplicaTimeout) / + math.Log(timeoutConfig.TimeoutAdjustmentFactor) + tc := Controller{ cfg: timeoutConfig, timeoutChannel: startChannel, + stopTicker: func() {}, + maxExponent: maxExponent, } return &tc } -func DefaultController() *Controller { - return NewController(DefaultConfig) +// Channel returns a channel that will receive the specific timeout. +// A new channel is created on each call of `StartTimeout`. +// Returns closed channel if no timer has been started. +func (t *Controller) Channel() <-chan time.Time { + return t.timeoutChannel } -// TimerInfo returns TimerInfo for the current timer. -// New struct is created for each timer. -// Is nil if no timer has been started. -func (t *Controller) TimerInfo() *model.TimerInfo { return t.timerInfo } +// StartTimeout starts the timeout of the specified type and returns the timer info +func (t *Controller) StartTimeout(ctx context.Context, view uint64) model.TimerInfo { + t.stopTicker() // stop old timeout -// Channel returns a channel that will receive the specific timeout. -// New channel is created for each timer. -// in the event the timeout is reached (specified as TimerInfo). -// returns closed channel if no timer has been started. -func (t *Controller) Channel() <-chan time.Time { return t.timeoutChannel } - -// StartTimeout starts the timeout of the specified type and returns the -func (t *Controller) StartTimeout(mode model.TimeoutMode, view uint64) *model.TimerInfo { - if t.timer != nil { // stop old timer - t.timer.Stop() - } - duration := t.computeTimeoutDuration(mode) + // setup new timer + durationMs := t.replicaTimeout() // duration of current view in units of Milliseconds + rebroadcastIntervalMs := math.Min(durationMs, t.cfg.MaxTimeoutObjectRebroadcastInterval) // time between attempted re-broadcast of timeouts if there is no progress + t.timeoutChannel = make(chan time.Time, 1) // channel for delivering timeouts - startTime := time.Now().UTC() - timer := time.NewTimer(duration) - timerInfo := model.TimerInfo{Mode: mode, View: view, StartTime: startTime, Duration: duration} - t.timer = timer - t.timeoutChannel = t.timer.C - t.timerInfo = &timerInfo + // start timeout logic for (re-)broadcasting timeout objects on regular basis as long as we are in the same round. + var childContext context.Context + childContext, t.stopTicker = context.WithCancel(ctx) + duration := time.Duration(durationMs) * time.Millisecond + rebroadcastInterval := time.Duration(rebroadcastIntervalMs) * time.Millisecond + go tickAfterTimeout(childContext, duration, rebroadcastInterval, t.timeoutChannel) - return &timerInfo + return model.TimerInfo{View: view, StartTime: time.Now().UTC(), Duration: duration} } -func (t *Controller) computeTimeoutDuration(mode model.TimeoutMode) time.Duration { - var duration time.Duration - switch mode { - case model.VoteCollectionTimeout: - duration = t.VoteCollectionTimeout() - case model.ReplicaTimeout: - duration = t.ReplicaTimeout() - default: - // This should never happen; Only protects code from future inconsistent modifications. - // There are only the two timeout modes explicitly handled above. Unless the enum - // containing the timeout mode is extended, the default case will never be reached. - panic("unknown timeout mode") +// tickAfterTimeout is a utility function which: +// 1. waits for the initial timeout and then sends the current time to `timeoutChannel` +// 2. and subsequently sends the current time every `tickInterval` to `timeoutChannel` +// +// If the receiver from the `timeoutChannel` falls behind and does not pick up the events, +// we drop ticks until the receiver catches up. When cancelling `ctx`, all timing logic stops. +// This approach allows to have a concurrent-safe implementation, where there is no unsafe state sharing between caller and +// ticking logic. +func tickAfterTimeout(ctx context.Context, duration time.Duration, tickInterval time.Duration, timeoutChannel chan<- time.Time) { + // wait for initial timeout + timer := time.NewTimer(duration) + select { + case t := <-timer.C: + timeoutChannel <- t // forward initial timeout to the sink + case <-ctx.Done(): + timer.Stop() // allows timer to be garbage collected (before it expires) + return } - return duration -} -// ReplicaTimeout returns the duration of the current view before we time out -func (t *Controller) ReplicaTimeout() time.Duration { - return time.Duration(t.cfg.ReplicaTimeout * 1e6) + // after we have reached the initial timeout, sent to `tickSink` every `tickInterval` until cancelled + ticker := time.NewTicker(tickInterval) + for { + select { + case t := <-ticker.C: + timeoutChannel <- t // forward ticks to the sink + case <-ctx.Done(): + ticker.Stop() // critical for ticker to be garbage collected + return + } + } } -// VoteCollectionTimeout returns the duration of Vote aggregation _after_ receiving a block -// during which the primary tries to aggregate votes for the view where it is leader -func (t *Controller) VoteCollectionTimeout() time.Duration { - // time.Duration expects an int64 as input which specifies the duration in units of nanoseconds (1E-9) - return time.Duration(t.cfg.ReplicaTimeout * 1e6 * t.cfg.VoteAggregationTimeoutFraction) +// replicaTimeout returns the duration of the current view in milliseconds before we time out +func (t *Controller) replicaTimeout() float64 { + if t.r <= t.cfg.HappyPathMaxRoundFailures { + return t.cfg.MinReplicaTimeout + } + r := float64(t.r - t.cfg.HappyPathMaxRoundFailures) + if r >= t.maxExponent { + return t.cfg.MaxReplicaTimeout + } + // compute timeout duration [in milliseconds]: + return t.cfg.MinReplicaTimeout * math.Pow(t.cfg.TimeoutAdjustmentFactor, r) } -// OnTimeout indicates to the Controller that the timeout was reached +// OnTimeout indicates to the Controller that a view change was triggered by a TC (unhappy path). func (t *Controller) OnTimeout() { - t.cfg.ReplicaTimeout = math.Min(t.cfg.ReplicaTimeout*t.cfg.TimeoutIncrease, timeoutCap) + if float64(t.r) >= t.maxExponent+float64(t.cfg.HappyPathMaxRoundFailures) { + return + } + t.r++ } // OnProgressBeforeTimeout indicates to the Controller that progress was made _before_ the timeout was reached func (t *Controller) OnProgressBeforeTimeout() { - t.cfg.ReplicaTimeout = math.Max(t.cfg.ReplicaTimeout*t.cfg.TimeoutDecrease, t.cfg.MinReplicaTimeout) + if t.r > 0 { + t.r-- + } } // BlockRateDelay is a delay to broadcast the proposal in order to control block production rate diff --git a/consensus/hotstuff/pacemaker/timeout/controller_test.go b/consensus/hotstuff/pacemaker/timeout/controller_test.go index 7158336332a..beb31f4eea9 100644 --- a/consensus/hotstuff/pacemaker/timeout/controller_test.go +++ b/consensus/hotstuff/pacemaker/timeout/controller_test.go @@ -10,41 +10,38 @@ import ( ) const ( - startRepTimeout float64 = 120 // Milliseconds - minRepTimeout float64 = 100 // Milliseconds - voteTimeoutFraction float64 = 0.5 // multiplicative factor - multiplicativeIncrease float64 = 1.5 // multiplicative factor - multiplicativeDecrease float64 = 0.85 // multiplicative factor + minRepTimeout float64 = 100 // Milliseconds + maxRepTimeout float64 = 10000 // Milliseconds + timeoutAdjustmentFactor float64 = 1.5 // timeout duration adjustment factor + happyPathMaxRoundFailures uint64 = 3 // number of failed rounds before increasing timeouts ) func initTimeoutController(t *testing.T) *Controller { tc, err := NewConfig( - time.Duration(startRepTimeout*1e6), time.Duration(minRepTimeout*1e6), - voteTimeoutFraction, - multiplicativeIncrease, - multiplicativeDecrease, - 0) + time.Duration(maxRepTimeout*1e6), + timeoutAdjustmentFactor, + happyPathMaxRoundFailures, + 0, + time.Duration(maxRepTimeout*1e6)) if err != nil { t.Fail() } return NewController(tc) } -// Test_TimeoutInitialization timeouts are initialized ands reported properly +// Test_TimeoutInitialization timeouts are initialized and reported properly func Test_TimeoutInitialization(t *testing.T) { tc := initTimeoutController(t) - assert.Equal(t, tc.ReplicaTimeout().Milliseconds(), int64(startRepTimeout)) - assert.Equal(t, tc.VoteCollectionTimeout().Milliseconds(), int64(startRepTimeout*voteTimeoutFraction)) + assert.Equal(t, tc.replicaTimeout(), minRepTimeout) - // verify that returned timeout channel + // verify that initially returned timeout channel is closed and `nil` is returned as `TimerInfo` select { case <-tc.Channel(): break default: assert.Fail(t, "timeout channel did not return") } - assert.True(t, tc.TimerInfo() == nil) tc.Channel() } @@ -52,15 +49,16 @@ func Test_TimeoutInitialization(t *testing.T) { func Test_TimeoutIncrease(t *testing.T) { tc := initTimeoutController(t) - for i := 1; i < 10; i += 1 { + // advance failed rounds beyond `happyPathMaxRoundFailures`; + for r := uint64(0); r < happyPathMaxRoundFailures; r++ { + tc.OnTimeout() + } + + for r := 1; r <= 10; r += 1 { tc.OnTimeout() assert.Equal(t, - tc.ReplicaTimeout().Milliseconds(), - int64(startRepTimeout*math.Pow(multiplicativeIncrease, float64(i))), - ) - assert.Equal(t, - tc.VoteCollectionTimeout().Milliseconds(), - int64(startRepTimeout*voteTimeoutFraction*math.Pow(multiplicativeIncrease, float64(i))), + tc.replicaTimeout(), + minRepTimeout*math.Pow(timeoutAdjustmentFactor, float64(r)), ) } } @@ -68,20 +66,19 @@ func Test_TimeoutIncrease(t *testing.T) { // Test_TimeoutDecrease verifies that timeout decreases exponentially func Test_TimeoutDecrease(t *testing.T) { tc := initTimeoutController(t) - tc.OnTimeout() - tc.OnTimeout() - tc.OnTimeout() - repTimeout := startRepTimeout * math.Pow(multiplicativeIncrease, 3.0) - for i := 1; i <= 6; i += 1 { + // failed rounds counter + r := uint64(0) + + // advance failed rounds beyond `happyPathMaxRoundFailures`; subsequent progress should reduce timeout again + for ; r <= happyPathMaxRoundFailures*2; r++ { + tc.OnTimeout() + } + for ; r > happyPathMaxRoundFailures; r-- { tc.OnProgressBeforeTimeout() assert.Equal(t, - tc.ReplicaTimeout().Milliseconds(), - int64(repTimeout*math.Pow(multiplicativeDecrease, float64(i))), - ) - assert.Equal(t, - tc.VoteCollectionTimeout().Milliseconds(), - int64((repTimeout*math.Pow(multiplicativeDecrease, float64(i)))*voteTimeoutFraction), + tc.replicaTimeout(), + minRepTimeout*math.Pow(timeoutAdjustmentFactor, float64(r-1-happyPathMaxRoundFailures)), ) } } @@ -90,36 +87,38 @@ func Test_TimeoutDecrease(t *testing.T) { func Test_MinCutoff(t *testing.T) { tc := initTimeoutController(t) - tc.OnTimeout() // replica timeout increases 120 -> 1.5 * 120 = 180 - tc.OnProgressBeforeTimeout() // replica timeout decreases 180 -> 180 * 0.85 = 153 - tc.OnProgressBeforeTimeout() // replica timeout decreases 153 -> 153 * 0.85 = 130.05 - tc.OnProgressBeforeTimeout() // replica timeout decreases 130.05 -> 130.05 * 0.85 = 110.5425 - tc.OnProgressBeforeTimeout() // replica timeout decreases 110.5425 -> max(110.5425 * 0.85, 100) = 100 + for r := uint64(0); r < happyPathMaxRoundFailures; r++ { + tc.OnTimeout() // replica timeout doesn't increase since r < happyPathMaxRoundFailures. + } + + tc.OnTimeout() // replica timeout increases 100 -> 3/2 * 100 = 150 + tc.OnTimeout() // replica timeout increases 150 -> 3/2 * 150 = 225 + tc.OnProgressBeforeTimeout() // replica timeout decreases 225 -> 180 * 2/3 = 150 + tc.OnProgressBeforeTimeout() // replica timeout decreases 150 -> 153 * 2/3 = 100 + tc.OnProgressBeforeTimeout() // replica timeout decreases 100 -> 100 * 2/3 = max(66.6, 100) = 100 tc.OnProgressBeforeTimeout() - assert.Equal(t, tc.ReplicaTimeout().Milliseconds(), int64(minRepTimeout)) - assert.Equal(t, tc.VoteCollectionTimeout().Milliseconds(), int64(minRepTimeout*voteTimeoutFraction)) + assert.Equal(t, tc.replicaTimeout(), minRepTimeout) } -// Test_MinCutoff verifies that timeout does not increase beyond timeout cap +// Test_MaxCutoff verifies that timeout does not increase beyond timeout cap func Test_MaxCutoff(t *testing.T) { - // here we use a different timeout controller with a larger timeoutIncrease to avoid too many iterations - c, err := NewConfig( - time.Duration(200*float64(time.Millisecond)), - time.Duration(minRepTimeout*float64(time.Millisecond)), - voteTimeoutFraction, - 10, - multiplicativeDecrease, - 0) - if err != nil { - t.Fail() - } - tc := NewController(c) + tc := initTimeoutController(t) + + // we update the following two values here in the test, which is a naive reference implementation + unboundedReferenceTimeout := minRepTimeout + r := -1 * int64(happyPathMaxRoundFailures) // only start increasing `unboundedReferenceTimeout` when this becomes positive - for i := 1; i <= 50; i += 1 { - tc.OnTimeout() // after already 7 iterations we should have reached the max value - assert.True(t, float64(tc.ReplicaTimeout().Milliseconds()) <= timeoutCap) - assert.True(t, float64(tc.VoteCollectionTimeout().Milliseconds()) <= timeoutCap*voteTimeoutFraction) + // add timeouts until our `unboundedReferenceTimeout` exceeds the limit + for { + tc.OnTimeout() + if r++; r > 0 { + unboundedReferenceTimeout *= timeoutAdjustmentFactor + } + if unboundedReferenceTimeout > maxRepTimeout { + assert.True(t, tc.replicaTimeout() <= maxRepTimeout) + return // end of test + } } } @@ -129,8 +128,8 @@ func Test_CombinedIncreaseDecreaseDynamics(t *testing.T) { increase, decrease := true, false testDynamicSequence := func(seq []bool) { tc := initTimeoutController(t) - var numberIncreases int = 0 - var numberDecreases int = 0 + tc.cfg.HappyPathMaxRoundFailures = 0 // set happy path rounds to zero to simplify calculation + numberIncreases, numberDecreases := 0, 0 for _, increase := range seq { if increase { numberIncreases += 1 @@ -141,26 +140,26 @@ func Test_CombinedIncreaseDecreaseDynamics(t *testing.T) { } } - expectedRepTimeout := startRepTimeout * math.Pow(multiplicativeIncrease, float64(numberIncreases)) * math.Pow(multiplicativeDecrease, float64(numberDecreases)) - numericalError := math.Abs(expectedRepTimeout - float64(tc.ReplicaTimeout().Milliseconds())) - require.True(t, numericalError <= 1.0) // at most one millisecond numerical error - numericalError = math.Abs(expectedRepTimeout*voteTimeoutFraction - float64(tc.VoteCollectionTimeout().Milliseconds())) - require.True(t, numericalError <= 1.0) // at most one millisecond numerical error + expectedRepTimeout := minRepTimeout * math.Pow(timeoutAdjustmentFactor, float64(numberIncreases-numberDecreases)) + numericalError := math.Abs(expectedRepTimeout - tc.replicaTimeout()) + require.LessOrEqual(t, numericalError, 1.0) // at most one millisecond numerical error } testDynamicSequence([]bool{increase, increase, increase, decrease, decrease, decrease}) testDynamicSequence([]bool{increase, decrease, increase, decrease, increase, decrease}) + testDynamicSequence([]bool{increase, increase, increase, increase, increase, decrease}) } +// Test_BlockRateDelay check that correct block rate delay is returned func Test_BlockRateDelay(t *testing.T) { - // here we use a different timeout controller with a larger timeoutIncrease to avoid too many iterations + c, err := NewConfig( - time.Duration(200*float64(time.Millisecond)), time.Duration(minRepTimeout*float64(time.Millisecond)), - voteTimeoutFraction, - 10, - multiplicativeDecrease, - time.Second) + time.Duration(maxRepTimeout*float64(time.Millisecond)), + timeoutAdjustmentFactor, + happyPathMaxRoundFailures, + time.Second, + time.Duration(maxRepTimeout*float64(time.Millisecond))) if err != nil { t.Fail() } diff --git a/consensus/hotstuff/persister.go b/consensus/hotstuff/persister.go index 45ba9686c5f..eaed7fcba57 100644 --- a/consensus/hotstuff/persister.go +++ b/consensus/hotstuff/persister.go @@ -3,16 +3,21 @@ package hotstuff // Persister is responsible for persisting state we need to bootstrap after a // restart or crash. type Persister interface { + // GetSafetyData will retrieve last persisted safety data. + // During normal operations, no errors are expected. + GetSafetyData() (*SafetyData, error) - // GetStarted will retrieve the last started view. - GetStarted() (uint64, error) + // PutSafetyData persists the last safety data. + // This method blocks until `safetyData` was successfully persisted. + // During normal operations, no errors are expected. + PutSafetyData(safetyData *SafetyData) error - // GetVoted will retrieve the last voted view. - GetVoted() (uint64, error) + // GetLivenessData will retrieve last persisted liveness data. + // During normal operations, no errors are expected. + GetLivenessData() (*LivenessData, error) - // PutStarted persists the last started view. - PutStarted(view uint64) error - - // PutVoted persists the last voted view. - PutVoted(view uint64) error + // PutLivenessData persists the last liveness data. + // This method blocks until `safetyData` was successfully persisted. + // During normal operations, no errors are expected. + PutLivenessData(livenessData *LivenessData) error } diff --git a/consensus/hotstuff/persister/persister.go b/consensus/hotstuff/persister/persister.go index 01c6300ffe1..f7b69575dac 100644 --- a/consensus/hotstuff/persister/persister.go +++ b/consensus/hotstuff/persister/persister.go @@ -3,17 +3,22 @@ package persister import ( "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/storage/badger/operation" ) // Persister can persist relevant information for hotstuff. +// Persister depends on protocol.State root snapshot bootstrapping to set initial values for +// SafetyData and LivenessData. These values must be initialized before first use of Persister. type Persister struct { db *badger.DB chainID flow.ChainID } -// New creates a nev persister using the injected stores to persist +var _ hotstuff.Persister = (*Persister)(nil) + +// New creates a new Persister using the injected data base to persist // relevant hotstuff data. func New(db *badger.DB, chainID flow.ChainID) *Persister { p := &Persister{ @@ -23,26 +28,30 @@ func New(db *badger.DB, chainID flow.ChainID) *Persister { return p } -// GetStarted returns the last persisted started view. -func (p *Persister) GetStarted() (uint64, error) { - var view uint64 - err := p.db.View(operation.RetrieveStartedView(p.chainID, &view)) - return view, err +// GetSafetyData will retrieve last persisted safety data. +// During normal operations, no errors are expected. +func (p *Persister) GetSafetyData() (*hotstuff.SafetyData, error) { + var safetyData hotstuff.SafetyData + err := p.db.View(operation.RetrieveSafetyData(p.chainID, &safetyData)) + return &safetyData, err } -// GetVoted returns the last persisted started view. -func (p *Persister) GetVoted() (uint64, error) { - var view uint64 - err := p.db.View(operation.RetrieveVotedView(p.chainID, &view)) - return view, err +// GetLivenessData will retrieve last persisted liveness data. +// During normal operations, no errors are expected. +func (p *Persister) GetLivenessData() (*hotstuff.LivenessData, error) { + var livenessData hotstuff.LivenessData + err := p.db.View(operation.RetrieveLivenessData(p.chainID, &livenessData)) + return &livenessData, err } -// PutStarted persists the view when we start it in hotstuff. -func (p *Persister) PutStarted(view uint64) error { - return operation.RetryOnConflict(p.db.Update, operation.UpdateStartedView(p.chainID, view)) +// PutSafetyData persists the last safety data. +// During normal operations, no errors are expected. +func (p *Persister) PutSafetyData(safetyData *hotstuff.SafetyData) error { + return operation.RetryOnConflict(p.db.Update, operation.UpdateSafetyData(p.chainID, safetyData)) } -// PutVoted persist the view when we voted in hotstuff. -func (p *Persister) PutVoted(view uint64) error { - return operation.RetryOnConflict(p.db.Update, operation.UpdateVotedView(p.chainID, view)) +// PutLivenessData persists the last liveness data. +// During normal operations, no errors are expected. +func (p *Persister) PutLivenessData(livenessData *hotstuff.LivenessData) error { + return operation.RetryOnConflict(p.db.Update, operation.UpdateLivenessData(p.chainID, livenessData)) } diff --git a/consensus/hotstuff/runner/single_runner.go b/consensus/hotstuff/runner/single_runner.go deleted file mode 100644 index 841222dec0f..00000000000 --- a/consensus/hotstuff/runner/single_runner.go +++ /dev/null @@ -1,78 +0,0 @@ -package runner - -import ( - "sync" -) - -// SingleRunner is a support struct for implementing module.ReadyDoneAware -type SingleRunner struct { - stateTransition sync.Mutex // lock for preventing concurrent state transitions - - startupCommenced bool // indicates whether Start(...) invoked - startupCompleted chan struct{} // channel is closed when startup was completed and the component is properly running - - shutdownCommenced bool // indicates whether Stop() invoked - shutdownSignal chan struct{} // used to signal that shutdown has commenced - - shutdownCompleted chan struct{} // used to signal that shutdown was completed and the component is done -} - -func NewSingleRunner() SingleRunner { - return SingleRunner{ - stateTransition: sync.Mutex{}, - startupCommenced: false, - startupCompleted: make(chan struct{}), - shutdownCommenced: false, - shutdownSignal: make(chan struct{}), - shutdownCompleted: make(chan struct{}), - } -} - -// ShutdownSignal returns a channel that is closed when the shutdown signal has been given -func (u *SingleRunner) ShutdownSignal() <-chan struct{} { - return u.shutdownSignal -} - -func (s *SingleRunner) Start(f func()) <-chan struct{} { - s.stateTransition.Lock() - if s.startupCommenced || s.shutdownCommenced { - s.stateTransition.Unlock() - return s.startupCompleted - } - - s.startupCommenced = true - go func() { - close(s.startupCompleted) - s.stateTransition.Unlock() - f() - // there are two cases f() would exit: - // (a) f exited on its own without Abort() being called (this is generally an internal error) - // (b) f exited as a reaction to Abort() being called - // In either case, we want to abort and close shutdownCompleted - s.stateTransition.Lock() - s.unsafeCommenceShutdown() - close(s.shutdownCompleted) - s.stateTransition.Unlock() - }() - - return s.startupCompleted -} - -// Abort() will abort the SingleRunner. Note that the channel returned from Start() will never be -// closed if the SingleRunner is aborted before Start() is called. This mimics the real world case: -// - consider a runner at a starting position of a race waiting for the start signal -// - if the runner to told to abort the race _before_ the start signal occurred, the runner will never start -func (s *SingleRunner) Abort() <-chan struct{} { - s.stateTransition.Lock() - s.unsafeCommenceShutdown() - s.stateTransition.Unlock() - return s.shutdownCompleted -} - -// unsafeCommenceShutdown executes the shutdown logic once but is not concurrency safe -func (s *SingleRunner) unsafeCommenceShutdown() { - if !s.shutdownCommenced { - s.shutdownCommenced = true - close(s.shutdownSignal) - } -} diff --git a/consensus/hotstuff/safety_rules.go b/consensus/hotstuff/safety_rules.go new file mode 100644 index 00000000000..d4db634c79a --- /dev/null +++ b/consensus/hotstuff/safety_rules.go @@ -0,0 +1,46 @@ +package hotstuff + +import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +type SafetyData struct { + // LockedOneChainView is the head block's view of the newest 1-chain this replica has voted for. + // The 1-chain can be indirect. + // <·· <QC>[B0] <- <QC_B0>[B1] <- [my vote for B1] + // In the depicted scenario, the replica voted for block B1, which forms a (potentially indirect) + // 1-chain on top of B0. The replica updated LockedOneChainView to the max of the current value and + // QC_B0.View = B0.View. Thereby, the safety module guarantees that the replica will not sign + // a TimeoutObject that would allow a malicious leader to fork below the latest finalized block. + LockedOneChainView uint64 + // HighestAcknowledgedView is the highest view where we have voted or triggered a timeout + HighestAcknowledgedView uint64 + // LastTimeout is the last timeout that was produced by this node (may be nil if no timeout occurred yet) + LastTimeout *model.TimeoutObject +} + +// SafetyRules enforces all consensus rules that guarantee safety. It produces votes for +// the given blocks or TimeoutObject for the given views, only if all safety rules are satisfied. +type SafetyRules interface { + // ProduceVote takes a block proposal and current view, and decides whether to vote for the block. + // Voting is deterministic meaning voting for same proposal will always result in the same vote. + // Returns: + // * (vote, nil): On the _first_ block for the current view that is safe to vote for. + // Subsequently, voter does _not_ vote for any _other_ block with the same (or lower) view. + // SafetyRules internally caches and persists its latest vote. As long as the SafetyRules' internal + // state remains unchanged, ProduceVote will return its cached for identical inputs. + // * (nil, model.NoVoteError): If the safety module decides that it is not safe to vote for the given block. + // This is a sentinel error and _expected_ during normal operation. + // All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). + ProduceVote(proposal *model.Proposal, curView uint64) (*model.Vote, error) + // ProduceTimeout takes current view, highest locally known QC and TC (optional, must be nil if and + // only if QC is for previous view) and decides whether to produce timeout for current view. + // Returns: + // * (timeout, nil): It is safe to timeout for current view using newestQC and lastViewTC. + // * (nil, model.NoTimeoutError): If replica is not part of the authorized consensus committee (anymore) and + // therefore is not authorized to produce a valid timeout object. This sentinel error is _expected_ during + // normal operation, e.g. during the grace-period after Epoch switchover or after the replica self-ejected. + // All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). + ProduceTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) +} diff --git a/consensus/hotstuff/safetyrules/safety_rules.go b/consensus/hotstuff/safetyrules/safety_rules.go new file mode 100644 index 00000000000..5ce5da8198a --- /dev/null +++ b/consensus/hotstuff/safetyrules/safety_rules.go @@ -0,0 +1,287 @@ +package safetyrules + +import ( + "fmt" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// SafetyRules is a dedicated module that enforces consensus safety. This component has the sole authority to generate +// votes and timeouts. It follows voting and timeout rules for creating votes and timeouts respectively. +// Caller can be sure that created vote or timeout doesn't break safety and can be used in consensus process. +// SafetyRules relies on hotstuff.Persister to store latest state of hotstuff.SafetyData. +// +// The voting rules implemented by SafetyRules are: +// 1. Replicas vote strictly in increasing rounds +// 2. Each block has to include a TC or a QC from the previous round. +// a. [Happy path] If the previous round resulted in a QC then new QC should extend it. +// b. [Recovery path] If the previous round did *not* result in a QC, the leader of the +// subsequent round *must* include a valid TC for the previous round in its block. +// +// NOT safe for concurrent use. +type SafetyRules struct { + signer hotstuff.Signer + persist hotstuff.Persister + committee hotstuff.DynamicCommittee // only produce votes when we are valid committee members + safetyData *hotstuff.SafetyData +} + +var _ hotstuff.SafetyRules = (*SafetyRules)(nil) + +// New creates a new SafetyRules instance +func New( + signer hotstuff.Signer, + persist hotstuff.Persister, + committee hotstuff.DynamicCommittee, +) (*SafetyRules, error) { + // get the last stored safety data + safetyData, err := persist.GetSafetyData() + if err != nil { + return nil, fmt.Errorf("could not recover safety data: %w", err) + } + + return &SafetyRules{ + signer: signer, + persist: persist, + committee: committee, + safetyData: safetyData, + }, nil +} + +// ProduceVote will make a decision on whether it will vote for the given proposal, the returned +// error indicates whether to vote or not. +// To ensure that only safe proposals are being voted on, we check that the proposer is a valid committee member and that the +// proposal complies with voting rules. +// We expect that only well-formed proposals with valid signatures are submitted for voting. +// The curView is taken as input to ensure SafetyRules will only vote for proposals at current view and prevent double voting. +// Returns: +// - (vote, nil): On the _first_ block for the current view that is safe to vote for. +// Subsequently, voter does _not_ vote for any other block with the same (or lower) view. +// - (nil, model.NoVoteError): If the voter decides that it does not want to vote for the given block. +// This is a sentinel error and _expected_ during normal operation. +// +// All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). +func (r *SafetyRules) ProduceVote(proposal *model.Proposal, curView uint64) (*model.Vote, error) { + block := proposal.Block + // sanity checks: + if curView != block.View { + return nil, fmt.Errorf("expecting block for current view %d, but block's view is %d", curView, block.View) + } + + err := r.IsSafeToVote(proposal) + if err != nil { + return nil, fmt.Errorf("not safe to vote for proposal %x: %w", proposal.Block.BlockID, err) + } + + // we expect that only valid proposals are submitted for voting + // we need to make sure that proposer is not ejected to decide to vote or not + _, err = r.committee.IdentityByBlock(block.BlockID, block.ProposerID) + if model.IsInvalidSignerError(err) { + // the proposer must be ejected since the proposal has already been validated, + // which ensures that the proposer was a valid committee member at the start of the epoch + return nil, model.NewNoVoteErrorf("proposer ejected: %w", err) + } + if err != nil { + return nil, fmt.Errorf("internal error retrieving Identity of proposer %x at block %x: %w", block.ProposerID, block.BlockID, err) + } + + // Do not produce a vote for blocks where we are not a valid committee member. + // HotStuff will ask for a vote for the first block of the next epoch, even if we + // have zero weight in the next epoch. Such vote can't be used to produce valid QCs. + _, err = r.committee.IdentityByBlock(block.BlockID, r.committee.Self()) + if model.IsInvalidSignerError(err) { + return nil, model.NewNoVoteErrorf("I am not authorized to vote for block %x: %w", block.BlockID, err) + } + if err != nil { + return nil, fmt.Errorf("could not get self identity: %w", err) + } + + vote, err := r.signer.CreateVote(block) + if err != nil { + return nil, fmt.Errorf("could not vote for block: %w", err) + } + + // vote for the current view has been produced, update safetyData + r.safetyData.HighestAcknowledgedView = curView + if r.safetyData.LockedOneChainView < block.QC.View { + r.safetyData.LockedOneChainView = block.QC.View + } + + err = r.persist.PutSafetyData(r.safetyData) + if err != nil { + return nil, fmt.Errorf("could not persist safety data: %w", err) + } + + return vote, nil +} + +// ProduceTimeout takes current view, highest locally known QC and TC (optional, must be nil if and +// only if QC is for previous view) and decides whether to produce timeout for current view. +// Returns: +// - (timeout, nil): It is safe to timeout for current view using newestQC and lastViewTC. +// - (nil, model.NoTimeoutError): If replica is not part of the authorized consensus committee (anymore) and +// therefore is not authorized to produce a valid timeout object. This sentinel error is _expected_ during +// normal operation, e.g. during the grace-period after Epoch switchover or after the replica self-ejected. +// +// All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). +func (r *SafetyRules) ProduceTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + lastTimeout := r.safetyData.LastTimeout + if lastTimeout != nil && lastTimeout.View == curView { + // model.TimeoutObject are conceptually immutable, hence we create a shallow copy here, which allows us to increment TimeoutTick + updatedTimeout := *lastTimeout + updatedTimeout.TimeoutTick += 1 + + // persist updated TimeoutObject in `safetyData` and return it + r.safetyData.LastTimeout = &updatedTimeout + err := r.persist.PutSafetyData(r.safetyData) + if err != nil { + return nil, fmt.Errorf("could not persist safety data: %w", err) + } + return r.safetyData.LastTimeout, nil + } + + err := r.IsSafeToTimeout(curView, newestQC, lastViewTC) + if err != nil { + return nil, fmt.Errorf("local, trusted inputs failed safety rules: %w", err) + } + + // Do not produce a timeout for view where we are not a valid committee member. + _, err = r.committee.IdentityByEpoch(curView, r.committee.Self()) + if err != nil { + if model.IsInvalidSignerError(err) { + return nil, model.NewNoTimeoutErrorf("I am not authorized to timeout for view %d: %w", curView, err) + } + return nil, fmt.Errorf("could not get self identity: %w", err) + } + + timeout, err := r.signer.CreateTimeout(curView, newestQC, lastViewTC) + if err != nil { + return nil, fmt.Errorf("could not create timeout at view %d: %w", curView, err) + } + + r.safetyData.HighestAcknowledgedView = curView + r.safetyData.LastTimeout = timeout + + err = r.persist.PutSafetyData(r.safetyData) + if err != nil { + return nil, fmt.Errorf("could not persist safety data: %w", err) + } + + return timeout, nil +} + +// IsSafeToVote checks if this proposal is valid in terms of voting rules, if voting for this proposal won't break safety rules. +// Expected errors during normal operations: +// - NoVoteError if replica already acted during this view (either voted or generated timeout) +func (r *SafetyRules) IsSafeToVote(proposal *model.Proposal) error { + blockView := proposal.Block.View + + err := r.validateEvidenceForEnteringView(blockView, proposal.Block.QC, proposal.LastViewTC) + if err != nil { + // As we are expecting the blocks to be pre-validated, any failure here is a symptom of an internal bug. + return fmt.Errorf("proposal failed consensus validity check") + } + + // This check satisfies voting rule 1 + // 1. Replicas vote strictly in increasing rounds, + // block's view must be greater than the view that we have voted for + acView := r.safetyData.HighestAcknowledgedView + if blockView == acView { + return model.NewNoVoteErrorf("already voted or generated timeout in view %d", blockView) + } + if blockView < acView { + return fmt.Errorf("already acted during view %d but got proposal for lower view %d", acView, blockView) + } + + return nil +} + +// IsSafeToTimeout checks if it's safe to timeout with proposed data, i.e. timing out won't break safety. +// newestQC is the valid QC with the greatest view that we have observed. +// lastViewTC is the TC for the previous view (might be nil). +// +// When generating a timeout, the inputs are provided by node-internal components. Failure to comply with +// the protocol is a symptom of an internal bug. We don't expect any errors during normal operations. +func (r *SafetyRules) IsSafeToTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) error { + err := r.validateEvidenceForEnteringView(curView, newestQC, lastViewTC) + if err != nil { + return fmt.Errorf("not safe to timeout: %w", err) + } + + if newestQC.View < r.safetyData.LockedOneChainView { + return fmt.Errorf("have already seen QC for view %d, but newest QC is reported to be for view %d", r.safetyData.LockedOneChainView, newestQC.View) + } + if curView+1 <= r.safetyData.HighestAcknowledgedView { + return fmt.Errorf("cannot generate timeout for past view %d", curView) + } + // the logic for rejecting inputs with `curView <= newestQC.View` is already contained + // in `validateEvidenceForEnteringView(..)`, because it only passes if + // * either `curView == newestQC.View + 1` (condition 2) + // * or `curView > newestQC.View` (condition 4) + + return nil +} + +// validateEvidenceForEnteringView performs the following check that is fundamental for consensus safety: +// Whenever a replica acts within a view, it must prove that is has sufficient evidence to enter this view +// Specifically: +// 1. The replica must always provide a QC and optionally a TC. +// 2. [Happy Path] If the previous round (i.e. `view -1`) resulted in a QC, the replica is allowed to transition to `view`. +// The QC from the previous round provides sufficient evidence. Furthermore, to prevent resource-exhaustion attacks, +// we require that no TC is included as part of the proof. +// 3. Following the Happy Path has priority over following the Recovery Path (specified below). +// 4. [Recovery Path] If the previous round (i.e. `view -1`) did *not* result in a QC, a TC from the previous round +// is required to transition to `view`. The following additional consistency requirements have to be satisfied: +// (a) newestQC.View + 1 < view +// Otherwise, the replica has violated condition 3 (in case newestQC.View + 1 = view); or the replica +// failed to apply condition 2 (in case newestQC.View + 1 > view). +// (b) newestQC.View ≥ lastViewTC.NewestQC.View +// Otherwise, the replica has violated condition 3. +// +// SafetyRules has the sole signing authority and enforces adherence to these conditions. In order to generate valid +// consensus signatures, the replica must provide the respective evidence (required QC + optional TC) to its +// internal SafetyRules component for each consensus action that the replica wants to take: +// - primary signing its own proposal +// - replica voting for a block +// - replica generating a timeout message +// +// During normal operations, no errors are expected: +// - As we are expecting the blocks to be pre-validated, any failure here is a symptom of an internal bug. +// - When generating a timeout, the inputs are provided by node-internal components. Failure to comply with +// the protocol is a symptom of an internal bug. +func (r *SafetyRules) validateEvidenceForEnteringView(view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) error { + // Condition 1: + if newestQC == nil { + return fmt.Errorf("missing the mandatory QC") + } + + // Condition 2: + if newestQC.View+1 == view { + if lastViewTC != nil { + return fmt.Errorf("when QC is for prior round, no TC should be provided") + } + return nil + } + // Condition 3: if we reach the following lines, the happy path is not satisfied. + + // Condition 4: + if lastViewTC == nil { + return fmt.Errorf("expecting TC because QC is not for prior view; but didn't get any TC") + } + if lastViewTC.View+1 != view { + return fmt.Errorf("neither QC (view %d) nor TC (view %d) allows to transition to view %d", newestQC.View, lastViewTC.View, view) + } + if newestQC.View >= view { + // Note: we need to enforce here that `newestQC.View + 1 < view`, i.e. we error for `newestQC.View+1 >= view` + // However, `newestQC.View+1 == view` is impossible, because otherwise we would have walked into condition 2. + // Hence, it suffices to error if `newestQC.View+1 > view`, which is identical to `newestQC.View >= view` + return fmt.Errorf("still at view %d, despite knowing a QC for view %d", view, newestQC.View) + } + if newestQC.View < lastViewTC.NewestQC.View { + return fmt.Errorf("failed to update newest QC (still at view %d) despite a newer QC (view %d) being included in TC", newestQC.View, lastViewTC.NewestQC.View) + } + + return nil +} diff --git a/consensus/hotstuff/safetyrules/safety_rules_test.go b/consensus/hotstuff/safetyrules/safety_rules_test.go new file mode 100644 index 00000000000..2c2d9cc201a --- /dev/null +++ b/consensus/hotstuff/safetyrules/safety_rules_test.go @@ -0,0 +1,716 @@ +package safetyrules + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestSafetyRules(t *testing.T) { + suite.Run(t, new(SafetyRulesTestSuite)) +} + +// SafetyRulesTestSuite is a test suite for testing SafetyRules related functionality. +// SafetyRulesTestSuite setups mocks for injected modules and creates hotstuff.SafetyData +// based on next configuration: +// R <- B[QC_R] <- P[QC_B] +// B.View = S.View + 1 +// B - bootstrapped block, we are creating SafetyRules at block B +// Based on this HighestAcknowledgedView = B.View and +type SafetyRulesTestSuite struct { + suite.Suite + + bootstrapBlock *model.Block + proposal *model.Proposal + proposerIdentity *flow.Identity + ourIdentity *flow.Identity + signer *mocks.Signer + persister *mocks.Persister + committee *mocks.DynamicCommittee + safetyData *hotstuff.SafetyData + safety *SafetyRules +} + +func (s *SafetyRulesTestSuite) SetupTest() { + s.ourIdentity = unittest.IdentityFixture() + s.signer = &mocks.Signer{} + s.persister = &mocks.Persister{} + s.committee = &mocks.DynamicCommittee{} + s.proposerIdentity = unittest.IdentityFixture() + + // bootstrap at random bootstrapBlock + s.bootstrapBlock = helper.MakeBlock(helper.WithBlockView(100)) + s.proposal = helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+1), + helper.WithBlockProposer(s.proposerIdentity.NodeID)), + )) + + s.committee.On("Self").Return(s.ourIdentity.NodeID).Maybe() + s.committee.On("IdentityByBlock", mock.Anything, s.ourIdentity.NodeID).Return(s.ourIdentity, nil).Maybe() + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.proposal.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + s.committee.On("IdentityByEpoch", mock.Anything, s.ourIdentity.NodeID).Return(s.ourIdentity, nil).Maybe() + + s.safetyData = &hotstuff.SafetyData{ + LockedOneChainView: s.bootstrapBlock.View, + HighestAcknowledgedView: s.bootstrapBlock.View, + } + + s.persister.On("GetSafetyData").Return(s.safetyData, nil).Once() + var err error + s.safety, err = New(s.signer, s.persister, s.committee) + require.NoError(s.T(), err) +} + +// TestProduceVote_ShouldVote test basic happy path scenario where we vote for first block after bootstrap +// and next view ended with TC +func (s *SafetyRulesTestSuite) TestProduceVote_ShouldVote() { + expectedSafetyData := &hotstuff.SafetyData{ + LockedOneChainView: s.proposal.Block.QC.View, + HighestAcknowledgedView: s.proposal.Block.View, + } + + expectedVote := makeVote(s.proposal.Block) + s.signer.On("CreateVote", s.proposal.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), expectedVote, vote) + + s.persister.AssertCalled(s.T(), "PutSafetyData", expectedSafetyData) + + // producing vote for same view yields an error since we have voted already for this view + otherVote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.True(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), otherVote) + + lastViewTC := helper.MakeTC( + helper.WithTCView(s.proposal.Block.View+1), + helper.WithTCNewestQC(s.proposal.Block.QC)) + + // voting on proposal where last view ended with TC + proposalWithTC := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.proposal.Block.View+2), + helper.WithBlockProposer(s.proposerIdentity.NodeID))), + helper.WithLastViewTC(lastViewTC)) + + expectedSafetyData = &hotstuff.SafetyData{ + LockedOneChainView: s.proposal.Block.QC.View, + HighestAcknowledgedView: proposalWithTC.Block.View, + } + + expectedVote = makeVote(proposalWithTC.Block) + s.signer.On("CreateVote", proposalWithTC.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + s.committee.On("IdentityByBlock", proposalWithTC.Block.BlockID, proposalWithTC.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + + vote, err = s.safety.ProduceVote(proposalWithTC, proposalWithTC.Block.View) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutSafetyData", expectedSafetyData) +} + +// TestProduceVote_IncludedQCHigherThanTCsQC checks specific scenario where previous round resulted in TC and leader +// knows about QC which is not part of TC and qc.View > tc.NewestQC.View. We want to allow this, in this case leader +// includes his QC into proposal satisfies next condition: Block.QC.View > lastViewTC.NewestQC.View +func (s *SafetyRulesTestSuite) TestProduceVote_IncludedQCHigherThanTCsQC() { + lastViewTC := helper.MakeTC( + helper.WithTCView(s.proposal.Block.View+1), + helper.WithTCNewestQC(s.proposal.Block.QC)) + + // voting on proposal where last view ended with TC + proposalWithTC := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.proposal.Block), + helper.WithBlockView(s.proposal.Block.View+2), + helper.WithBlockProposer(s.proposerIdentity.NodeID))), + helper.WithLastViewTC(lastViewTC)) + + expectedSafetyData := &hotstuff.SafetyData{ + LockedOneChainView: proposalWithTC.Block.QC.View, + HighestAcknowledgedView: proposalWithTC.Block.View, + } + + require.Greater(s.T(), proposalWithTC.Block.QC.View, proposalWithTC.LastViewTC.NewestQC.View, + "for this test case we specifically require that qc.View > lastViewTC.NewestQC.View") + + expectedVote := makeVote(proposalWithTC.Block) + s.signer.On("CreateVote", proposalWithTC.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + s.committee.On("IdentityByBlock", proposalWithTC.Block.BlockID, proposalWithTC.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + + vote, err := s.safety.ProduceVote(proposalWithTC, proposalWithTC.Block.View) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutSafetyData", expectedSafetyData) +} + +// TestProduceVote_UpdateLockedOneChainView tests that LockedOneChainView is updated when sees a higher QC. +// Note: `LockedOneChainView` is only updated when the replica votes. +func (s *SafetyRulesTestSuite) TestProduceVote_UpdateLockedOneChainView() { + s.safety.safetyData.LockedOneChainView = 0 + + require.NotEqual(s.T(), s.safety.safetyData.LockedOneChainView, s.proposal.Block.QC.View, + "in this test LockedOneChainView is lower so it needs to be updated") + + expectedSafetyData := &hotstuff.SafetyData{ + LockedOneChainView: s.proposal.Block.QC.View, + HighestAcknowledgedView: s.proposal.Block.View, + } + + expectedVote := makeVote(s.proposal.Block) + s.signer.On("CreateVote", s.proposal.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutSafetyData", expectedSafetyData) +} + +// TestProduceVote_InvalidCurrentView tests that no vote is created if `curView` has invalid values. +// In particular, `SafetyRules` requires that: +// - the block's view matches `curView` +// - that values for `curView` are monotonously increasing +// +// Failing any of these conditions is a symptom of an internal bug; hence `SafetyRules` should +// _not_ return a `NoVoteError`. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidCurrentView() { + + s.Run("block-view-does-not-match", func() { + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View+1) + require.Nil(s.T(), vote) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + }) + s.Run("view-not-monotonously-increasing", func() { + // create block with view < HighestAcknowledgedView + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + func(block *model.Block) { + block.QC = helper.MakeQC(helper.WithQCView(s.safetyData.HighestAcknowledgedView - 2)) + }, + helper.WithBlockView(s.safetyData.HighestAcknowledgedView-1)))) + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Nil(s.T(), vote) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + }) + + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_NodeEjected tests that no vote is created if block proposer is ejected +func (s *SafetyRulesTestSuite) TestProduceVote_ProposerEjected() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.proposal.Block.ProposerID).Return(nil, model.NewInvalidSignerErrorf("node-ejected")).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.True(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_InvalidProposerIdentity tests that no vote is created if there was an exception retrieving proposer identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidProposerIdentity() { + *s.committee = mocks.DynamicCommittee{} + exception := errors.New("invalid-signer-identity") + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.proposal.Block.ProposerID).Return(nil, exception).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_NodeNotAuthorizedToVote tests that no vote is created if the voter is not authorized to vote. +// Nodes have zero weight in the grace periods around the epochs where they are authorized to participate. +// We don't want zero-weight nodes to vote in the first place, to avoid unnecessary traffic. +// Note: this also covers ejected nodes. In both cases, the committee will return an `InvalidSignerError`. +func (s *SafetyRulesTestSuite) TestProduceVote_NodeEjected() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity.NodeID) + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.ourIdentity.NodeID).Return(nil, model.NewInvalidSignerErrorf("node-ejected")).Once() + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.proposal.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.True(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_InvalidVoterIdentity tests that no vote is created if there was an exception retrieving voter identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidVoterIdentity() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity.NodeID) + exception := errors.New("invalid-signer-identity") + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.proposal.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + s.committee.On("IdentityByBlock", s.proposal.Block.BlockID, s.ourIdentity.NodeID).Return(nil, exception).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_CreateVoteException tests that no vote is created if vote creation raised an exception +func (s *SafetyRulesTestSuite) TestProduceVote_CreateVoteException() { + exception := errors.New("create-vote-exception") + s.signer.On("CreateVote", s.proposal.Block).Return(nil, exception).Once() + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_PersistStateException tests that no vote is created if persisting state failed +func (s *SafetyRulesTestSuite) TestProduceVote_PersistStateException() { + exception := errors.New("persister-exception") + s.persister.On("PutSafetyData", mock.Anything).Return(exception) + + vote := makeVote(s.proposal.Block) + s.signer.On("CreateVote", s.proposal.Block).Return(vote, nil).Once() + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) +} + +// TestProduceVote_VotingOnInvalidProposals tests different scenarios where we try to vote on unsafe blocks +// SafetyRules contain a variety of checks to confirm that QC and TC have the desired relationship to each other. +// In particular, we test: +// +// (i) A TC should be included in a proposal, if and only of the QC is not the prior view. +// (ii) When the proposal includes a TC (i.e. the QC not being for the prior view), the TC must be for the prior view. +// (iii) The QC in the block must have a smaller view than the block. +// (iv) If the block contains a TC, the TC cannot contain a newer QC than the block itself. +// +// Conditions (i) - (iv) are validity requirements for the block and all blocks that SafetyRules processes +// are supposed to be pre-validated. Hence, failing any of those conditions means we have an internal bug. +// Consequently, we expect SafetyRules to return exceptions but _not_ `NoVoteError`, because the latter +// indicates that the input block was valid, but we didn't want to vote. +func (s *SafetyRulesTestSuite) TestProduceVote_VotingOnInvalidProposals() { + + // a proposal which includes a QC for the previous round should not contain a TC + s.Run("proposal-includes-last-view-qc-and-tc", func() { + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+1))), + helper.WithLastViewTC(helper.MakeTC())) + s.committee.On("IdentityByBlock", proposal.Block.BlockID, proposal.Block.ProposerID).Return(s.proposerIdentity, nil).Maybe() + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("no-last-view-tc", func() { + // create block where Block.View != Block.QC.View+1 and LastViewTC = nil + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+2)))) + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("last-view-tc-invalid-view", func() { + // create block where Block.View != Block.QC.View+1 and + // Block.View != LastViewTC.View+1 + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+2))), + helper.WithLastViewTC( + helper.MakeTC( + helper.WithTCView(s.bootstrapBlock.View)))) + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("proposal-includes-QC-for-higher-view", func() { + // create block where Block.View != Block.QC.View+1 and + // Block.View == LastViewTC.View+1 and Block.QC.View >= Block.View + // in this case block is not safe to extend since proposal includes QC which is newer than the proposal itself. + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+2), + func(block *model.Block) { + block.QC = helper.MakeQC(helper.WithQCView(s.bootstrapBlock.View + 10)) + })), + helper.WithLastViewTC( + helper.MakeTC( + helper.WithTCView(s.bootstrapBlock.View+1)))) + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("last-view-tc-invalid-highest-qc", func() { + // create block where Block.View != Block.QC.View+1 and + // Block.View == LastViewTC.View+1 and Block.QC.View < LastViewTC.NewestQC.View + // in this case block is not safe to extend since proposal is built on top of QC, which is lower + // than QC presented in LastViewTC. + TONewestQC := helper.MakeQC(helper.WithQCView(s.bootstrapBlock.View + 1)) + proposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+2))), + helper.WithLastViewTC( + helper.MakeTC( + helper.WithTCView(s.bootstrapBlock.View+1), + helper.WithTCNewestQC(TONewestQC)))) + vote, err := s.safety.ProduceVote(proposal, proposal.Block.View) + require.Error(s.T(), err) + require.False(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + + s.signer.AssertNotCalled(s.T(), "CreateVote") + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceVote_VoteEquivocation tests scenario when we try to vote twice in same view. We require that replica +// follows next rules: +// - replica votes once per view +// - replica votes in monotonously increasing views +// +// Voting twice per round on equivocating proposals is considered a byzantine behavior. +// Expect a `model.NoVoteError` sentinel in such scenario. +func (s *SafetyRulesTestSuite) TestProduceVote_VoteEquivocation() { + expectedVote := makeVote(s.proposal.Block) + s.signer.On("CreateVote", s.proposal.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", mock.Anything).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.Block.View) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), expectedVote, vote) + + equivocatingProposal := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithParentBlock(s.bootstrapBlock), + helper.WithBlockView(s.bootstrapBlock.View+1), + helper.WithBlockProposer(s.proposerIdentity.NodeID)), + )) + + // voting at same view(event different proposal) should result in NoVoteError + vote, err = s.safety.ProduceVote(equivocatingProposal, s.proposal.Block.View) + require.True(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) +} + +// TestProduceVote_AfterTimeout tests a scenario where we first timeout for view and then try to produce a vote for +// same view, this should result in error since producing a timeout means that we have given up on this view +// and are in process of moving forward, no vote should be created. +func (s *SafetyRulesTestSuite) TestProduceVote_AfterTimeout() { + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + expectedTimeout := &model.TimeoutObject{ + View: view, + NewestQC: newestQC, + } + s.signer.On("CreateTimeout", view, newestQC, (*flow.TimeoutCertificate)(nil)).Return(expectedTimeout, nil).Once() + s.persister.On("PutSafetyData", mock.Anything).Return(nil).Once() + + // first timeout, then try to vote + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.NoError(s.T(), err) + require.NotNil(s.T(), timeout) + + // voting in same view after producing timeout is not allowed + vote, err := s.safety.ProduceVote(s.proposal, view) + require.True(s.T(), model.IsNoVoteError(err)) + require.Nil(s.T(), vote) + + s.signer.AssertExpectations(s.T()) + s.persister.AssertExpectations(s.T()) +} + +// TestProduceTimeout_ShouldTimeout tests that we can produce timeout in cases where +// last view was successful or not. Also tests last timeout caching. +func (s *SafetyRulesTestSuite) TestProduceTimeout_ShouldTimeout() { + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + expectedTimeout := &model.TimeoutObject{ + View: view, + NewestQC: newestQC, + } + + expectedSafetyData := &hotstuff.SafetyData{ + LockedOneChainView: s.safetyData.LockedOneChainView, + HighestAcknowledgedView: view, + LastTimeout: expectedTimeout, + } + s.signer.On("CreateTimeout", view, newestQC, (*flow.TimeoutCertificate)(nil)).Return(expectedTimeout, nil).Once() + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.NoError(s.T(), err) + require.Equal(s.T(), expectedTimeout, timeout) + + s.persister.AssertCalled(s.T(), "PutSafetyData", expectedSafetyData) + + // producing timeout with same arguments should return cached version but with incremented timeout tick + expectedSafetyData.LastTimeout = &model.TimeoutObject{} + *expectedSafetyData.LastTimeout = *expectedTimeout + expectedSafetyData.LastTimeout.TimeoutTick++ + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + + otherTimeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.NoError(s.T(), err) + require.Equal(s.T(), timeout.ID(), otherTimeout.ID()) + require.Equal(s.T(), timeout.TimeoutTick+1, otherTimeout.TimeoutTick) + + // to create new TO we need to provide a TC + lastViewTC := helper.MakeTC(helper.WithTCView(view), + helper.WithTCNewestQC(newestQC)) + + expectedTimeout = &model.TimeoutObject{ + View: view + 1, + NewestQC: newestQC, + LastViewTC: lastViewTC, + } + s.signer.On("CreateTimeout", view+1, newestQC, lastViewTC).Return(expectedTimeout, nil).Once() + expectedSafetyData = &hotstuff.SafetyData{ + LockedOneChainView: s.safetyData.LockedOneChainView, + HighestAcknowledgedView: view + 1, + LastTimeout: expectedTimeout, + } + s.persister.On("PutSafetyData", expectedSafetyData).Return(nil).Once() + + // creating new timeout should invalidate cache + otherTimeout, err = s.safety.ProduceTimeout(view+1, newestQC, lastViewTC) + require.NoError(s.T(), err) + require.NotNil(s.T(), otherTimeout) +} + +// TestProduceTimeout_NotSafeToTimeout tests that we don't produce a timeout when it's not safe +// We expect that the EventHandler to feed only request timeouts for the current view, providing valid set of inputs. +// Hence, the cases tested here would be symptoms of an internal bugs, and therefore should not result in an NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceTimeout_NotSafeToTimeout() { + + s.Run("newest-qc-nil", func() { + // newestQC cannot be nil + timeout, err := s.safety.ProduceTimeout(s.safetyData.LockedOneChainView, nil, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + // if a QC for the previous view is provided, a last view TC is unnecessary and must not be provided + s.Run("includes-last-view-qc-and-tc", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + + // tc not needed but included + timeout, err := s.safety.ProduceTimeout(newestQC.View+1, newestQC, helper.MakeTC()) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-view-tc-nil", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + + // tc needed but not included + timeout, err := s.safety.ProduceTimeout(newestQC.View+2, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-view-tc-for-wrong-view", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + // lastViewTC should be for newestQC.View+1 + lastViewTC := helper.MakeTC(helper.WithTCView(newestQC.View)) + + timeout, err := s.safety.ProduceTimeout(newestQC.View+2, newestQC, lastViewTC) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-view-equal-to-highest-QC", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + lastViewTC := helper.MakeTC(helper.WithTCView(s.safetyData.LockedOneChainView - 1)) + + timeout, err := s.safety.ProduceTimeout(s.safetyData.LockedOneChainView, newestQC, lastViewTC) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-view-below-highest-QC", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + lastViewTC := helper.MakeTC(helper.WithTCView(newestQC.View - 2)) + + timeout, err := s.safety.ProduceTimeout(newestQC.View-1, newestQC, lastViewTC) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-view-tc-is-newer", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + // newest QC included in TC cannot be higher than the newest QC known to replica + lastViewTC := helper.MakeTC(helper.WithTCView(newestQC.View+1), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(newestQC.View+1)))) + + timeout, err := s.safety.ProduceTimeout(newestQC.View+2, newestQC, lastViewTC) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("highest-qc-below-locked-round", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView - 1)) + + timeout, err := s.safety.ProduceTimeout(newestQC.View+1, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-view-below-highest-acknowledged-view", func() { + newestQC := helper.MakeQC(helper.WithQCView(s.safetyData.LockedOneChainView)) + // modify highest acknowledged view in a way that it's definitely bigger than the newest QC view + s.safetyData.HighestAcknowledgedView = newestQC.View + 10 + + timeout, err := s.safety.ProduceTimeout(newestQC.View+1, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + + s.signer.AssertNotCalled(s.T(), "CreateTimeout") + s.signer.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceTimeout_CreateTimeoutException tests that no timeout is created if timeout creation raised an exception +func (s *SafetyRulesTestSuite) TestProduceTimeout_CreateTimeoutException() { + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + + exception := errors.New("create-timeout-exception") + s.signer.On("CreateTimeout", view, newestQC, (*flow.TimeoutCertificate)(nil)).Return(nil, exception).Once() + vote, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceTimeout_PersistStateException tests that no timeout is created if persisting state failed +func (s *SafetyRulesTestSuite) TestProduceTimeout_PersistStateException() { + exception := errors.New("persister-exception") + s.persister.On("PutSafetyData", mock.Anything).Return(exception) + + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + expectedTimeout := &model.TimeoutObject{ + View: view, + NewestQC: newestQC, + } + + s.signer.On("CreateTimeout", view, newestQC, (*flow.TimeoutCertificate)(nil)).Return(expectedTimeout, nil).Once() + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.Nil(s.T(), timeout) + require.ErrorIs(s.T(), err, exception) +} + +// TestProduceTimeout_AfterVote tests a case where we first produce a vote and then try to timeout +// for same view. This behavior is expected and should result in valid timeout without any errors. +func (s *SafetyRulesTestSuite) TestProduceTimeout_AfterVote() { + expectedVote := makeVote(s.proposal.Block) + s.signer.On("CreateVote", s.proposal.Block).Return(expectedVote, nil).Once() + s.persister.On("PutSafetyData", mock.Anything).Return(nil).Times(2) + + view := s.proposal.Block.View + + // first produce vote, then try to timeout + vote, err := s.safety.ProduceVote(s.proposal, view) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + + expectedTimeout := &model.TimeoutObject{ + View: view, + NewestQC: newestQC, + } + + s.signer.On("CreateTimeout", view, newestQC, (*flow.TimeoutCertificate)(nil)).Return(expectedTimeout, nil).Once() + + // timing out for same view should be possible + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.NoError(s.T(), err) + require.NotNil(s.T(), timeout) + + s.persister.AssertExpectations(s.T()) + s.signer.AssertExpectations(s.T()) +} + +// TestProduceTimeout_InvalidProposerIdentity tests that no timeout is created if there was an exception retrieving proposer identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected model.NoTimeoutError. +func (s *SafetyRulesTestSuite) TestProduceTimeout_InvalidProposerIdentity() { + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + *s.committee = mocks.DynamicCommittee{} + exception := errors.New("invalid-signer-identity") + s.committee.On("IdentityByEpoch", view, s.ourIdentity.NodeID).Return(nil, exception).Once() + s.committee.On("Self").Return(s.ourIdentity.NodeID) + + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.Nil(s.T(), timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsNoTimeoutError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +// TestProduceTimeout_NodeEjected tests that no timeout is created if the replica is not authorized to create timeout. +// Nodes have zero weight in the grace periods around the epochs where they are authorized to participate. +// We don't want zero-weight nodes to participate in the first place, to avoid unnecessary traffic. +// Note: this also covers ejected nodes. In both cases, the committee will return an `InvalidSignerError`. +func (s *SafetyRulesTestSuite) TestProduceTimeout_NodeEjected() { + view := s.proposal.Block.View + newestQC := helper.MakeQC(helper.WithQCView(view - 1)) + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity.NodeID) + s.committee.On("IdentityByEpoch", view, s.ourIdentity.NodeID).Return(nil, model.NewInvalidSignerErrorf("")).Maybe() + + timeout, err := s.safety.ProduceTimeout(view, newestQC, nil) + require.Nil(s.T(), timeout) + require.True(s.T(), model.IsNoTimeoutError(err)) + s.persister.AssertNotCalled(s.T(), "PutSafetyData") +} + +func makeVote(block *model.Block) *model.Vote { + return &model.Vote{ + BlockID: block.BlockID, + View: block.View, + SigData: nil, // signature doesn't matter in this test case + } +} diff --git a/consensus/hotstuff/signature.go b/consensus/hotstuff/signature.go index bdb4a40b878..02a266fa95b 100644 --- a/consensus/hotstuff/signature.go +++ b/consensus/hotstuff/signature.go @@ -89,6 +89,55 @@ type WeightedSignatureAggregator interface { Aggregate() (flow.IdentifierList, []byte, error) } +// TimeoutSignatureAggregator aggregates timeout signatures for one particular view. +// When instantiating a TimeoutSignatureAggregator, the following information is supplied: +// - The view for which the aggregator collects timeouts. +// - For each replicas that is authorized to send a timeout at this particular view: +// the node ID, public staking keys, and weight +// +// Timeouts for other views or from non-authorized replicas are rejected. +// In their TimeoutObjects, replicas include a signature over the pair (view, newestQCView), +// where `view` is the view number the timeout is for and `newestQCView` is the view of +// the newest QC known to the replica. TimeoutSignatureAggregator collects these signatures, +// internally tracks the total weight of all collected signatures. Note that in general the +// signed messages are different, which makes the aggregation a comparatively expensive operation. +// Upon calling `Aggregate`, the TimeoutSignatureAggregator aggregates all valid signatures collected +// up to this point. The aggregate signature is guaranteed to be correct, as only valid signatures +// are excepted as inputs. +// TimeoutSignatureAggregator internally tracks the total weight of all collected signatures. +// Implementations must be concurrency safe. +type TimeoutSignatureAggregator interface { + // VerifyAndAdd verifies the signature under the stored public keys and adds the signature and the corresponding + // highest QC to the internal set. Internal set and collected weight is modified iff signature _is_ valid. + // The total weight of all collected signatures (excluding duplicates) is returned regardless + // of any returned error. + // Expected errors during normal operations: + // - model.InvalidSignerError if signerID is invalid (not a consensus participant) + // - model.DuplicatedSignerError if the signer has been already added + // - model.ErrInvalidSignature if signerID is valid but signature is cryptographically invalid + VerifyAndAdd(signerID flow.Identifier, sig crypto.Signature, newestQCView uint64) (totalWeight uint64, exception error) + + // TotalWeight returns the total weight presented by the collected signatures. + TotalWeight() uint64 + + // View returns the view that this instance is aggregating signatures for. + View() uint64 + + // Aggregate aggregates the signatures and returns with additional data. + // Aggregated signature will be returned as SigData of timeout certificate. + // Caller can be sure that resulting signature is valid. + // Expected errors during normal operations: + // - model.InsufficientSignaturesError if no signatures have been added yet + Aggregate() (signersInfo []TimeoutSignerInfo, aggregatedSig crypto.Signature, exception error) +} + +// TimeoutSignerInfo is a helper structure that stores the QC views that each signer +// contributed to a TC. Used as result of TimeoutSignatureAggregator.Aggregate() +type TimeoutSignerInfo struct { + NewestQCView uint64 + Signer flow.Identifier +} + // BlockSignatureData is an intermediate struct for Packer to pack the // aggregated signature data into raw bytes or unpack from raw bytes. type BlockSignatureData struct { @@ -102,11 +151,11 @@ type BlockSignatureData struct { // Packer packs aggregated signature data into raw bytes to be used in block header. type Packer interface { // Pack serializes the provided BlockSignatureData into a precursor format of a QC. - // blockID is the block that the aggregated signature is for. + // view is the view of the block that the aggregated signature is for. // sig is the aggregated signature data. // Expected error returns during normal operations: // * none; all errors are symptoms of inconsistent input data or corrupted internal state. - Pack(blockID flow.Identifier, sig *BlockSignatureData) (signerIndices []byte, sigData []byte, err error) + Pack(view uint64, sig *BlockSignatureData) (signerIndices []byte, sigData []byte, err error) // Unpack de-serializes the provided signature data. // sig is the aggregated signature data diff --git a/consensus/hotstuff/signature/block_signer_decoder.go b/consensus/hotstuff/signature/block_signer_decoder.go index cf2f8e483be..95fafcd688d 100644 --- a/consensus/hotstuff/signature/block_signer_decoder.go +++ b/consensus/hotstuff/signature/block_signer_decoder.go @@ -5,28 +5,29 @@ import ( "fmt" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/signature" - "github.com/onflow/flow-go/state" - "github.com/onflow/flow-go/storage" ) -// BlockSignerDecoder is a wrapper around the `hotstuff.Committee`, which implements +// BlockSignerDecoder is a wrapper around the `hotstuff.DynamicCommittee`, which implements // the auxilluary logic for de-coding signer indices of a block (header) to full node IDs type BlockSignerDecoder struct { - // TODO: update to Replicas API once active PaceMaker is merged - hotstuff.Committee + hotstuff.DynamicCommittee } -func NewBlockSignerDecoder(committee hotstuff.Committee) *BlockSignerDecoder { +func NewBlockSignerDecoder(committee hotstuff.DynamicCommittee) *BlockSignerDecoder { return &BlockSignerDecoder{committee} } var _ hotstuff.BlockSignerDecoder = (*BlockSignerDecoder)(nil) // DecodeSignerIDs decodes the signer indices from the given block header into full node IDs. +// Note: A block header contains a quorum certificate for its parent, which proves that the +// consensus committee has reached agreement on validity of parent block. Consequently, the +// returned IdentifierList contains the consensus participants that signed the parent block. // Expected Error returns during normal operations: -// - state.UnknownBlockError if block has not been ingested yet +// - model.ErrViewForUnknownEpoch if the given block's parent is within an unknown epoch // - signature.InvalidSignerIndicesError if signer indices included in the header do // not encode a valid subset of the consensus committee func (b *BlockSignerDecoder) DecodeSignerIDs(header *flow.Header) (flow.IdentifierList, error) { @@ -35,16 +36,12 @@ func (b *BlockSignerDecoder) DecodeSignerIDs(header *flow.Header) (flow.Identifi return []flow.Identifier{}, nil } - // The block header contains the signatures for the parents. Hence, we need to get the - // identities that were authorized to sign the parent block, to decode the signer indices. - members, err := b.Identities(header.ParentID) + members, err := b.IdentitiesByEpoch(header.ParentView) if err != nil { - // TODO: this potentially needs to be updated when we implement and document proper error handling for - // `hotstuff.Committee` and underlying code (such as `protocol.Snapshot`) - if errors.Is(err, storage.ErrNotFound) { - return nil, state.WrapAsUnknownBlockError(header.ID(), err) + if errors.Is(err, model.ErrViewForUnknownEpoch) { + return nil, fmt.Errorf("could not retrieve consensus participants for view %d: %w", header.ParentView, err) } - return nil, fmt.Errorf("fail to retrieve identities for block %v: %w", header.ID(), err) + return nil, fmt.Errorf("unexpected error retrieving identities for block %v: %w", header.ID(), err) } signerIDs, err := signature.DecodeSignerIndicesToIdentifiers(members.NodeIDs(), header.ParentVoterIndices) if err != nil { diff --git a/consensus/hotstuff/signature/block_signer_decoder_test.go b/consensus/hotstuff/signature/block_signer_decoder_test.go index 021425a3a0c..4325b50c7b7 100644 --- a/consensus/hotstuff/signature/block_signer_decoder_test.go +++ b/consensus/hotstuff/signature/block_signer_decoder_test.go @@ -9,11 +9,10 @@ import ( "github.com/stretchr/testify/suite" hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/order" "github.com/onflow/flow-go/module/signature" - "github.com/onflow/flow-go/state" - "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/unittest" ) @@ -24,7 +23,7 @@ func TestBlockSignerDecoder(t *testing.T) { type blockSignerDecoderSuite struct { suite.Suite allConsensus flow.IdentityList - committee *hotstuff.Committee + committee *hotstuff.DynamicCommittee decoder *BlockSignerDecoder block flow.Block @@ -34,16 +33,16 @@ func (s *blockSignerDecoderSuite) SetupTest() { // the default header fixture creates signerIDs for a committee of 10 nodes, so we prepare a committee same as that s.allConsensus = unittest.IdentityListFixture(40, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) + // mock consensus committee + s.committee = hotstuff.NewDynamicCommittee(s.T()) + s.committee.On("IdentitiesByEpoch", mock.Anything).Return(s.allConsensus, nil).Maybe() + // prepare valid test block: voterIndices, err := signature.EncodeSignersToIndices(s.allConsensus.NodeIDs(), s.allConsensus.NodeIDs()) require.NoError(s.T(), err) s.block = unittest.BlockFixture() s.block.Header.ParentVoterIndices = voterIndices - // mock consensus committee - s.committee = new(hotstuff.Committee) - s.committee.On("Identities", s.block.Header.ParentID).Return(s.allConsensus, nil) - s.decoder = NewBlockSignerDecoder(s.committee) } @@ -65,34 +64,34 @@ func (s *blockSignerDecoderSuite) Test_RootBlock() { require.Empty(s.T(), ids) } -// Test_UnknownBlock tests handling of an unknwon block. -// At the moment, hotstuff.Committee returns an storage.ErrNotFound for an unknown block, -// which we expect the `BlockSignerDecoder` to wrap into a `state.UnknownBlockError` -func (s *blockSignerDecoderSuite) Test_UnknownBlock() { - *s.committee = hotstuff.Committee{} - s.committee.On("Identities", mock.Anything).Return(nil, storage.ErrNotFound) +// Test_UnexpectedCommitteeException verifies that `BlockSignerDecoder` +// does _not_ erroneously interpret an unexpected exception from the committee as +// a sign of an unknown block, i.e. the decoder should _not_ return an `model.ErrViewForUnknownEpoch` or `signature.InvalidSignerIndicesError` +func (s *blockSignerDecoderSuite) Test_UnexpectedCommitteeException() { + exception := errors.New("unexpected exception") + *s.committee = *hotstuff.NewDynamicCommittee(s.T()) + s.committee.On("IdentitiesByEpoch", mock.Anything).Return(nil, exception) ids, err := s.decoder.DecodeSignerIDs(s.block.Header) require.Empty(s.T(), ids) - require.True(s.T(), state.IsUnknownBlockError(err)) + require.NotErrorIs(s.T(), err, model.ErrViewForUnknownEpoch) + require.False(s.T(), signature.IsInvalidSignerIndicesError(err)) + require.True(s.T(), errors.Is(err, exception)) } -// Test_UnexpectedCommitteeException verifies that `BlockSignerDecoder` -// does _not_ erroneously interpret an unexpecgted exception from the committee as -// a sign of an unknown block, i.e. the decouder should _not_ return an `state.UnknownBlockError` -func (s *blockSignerDecoderSuite) Test_UnexpectedCommitteeException() { - exception := errors.New("unexpected exception") - *s.committee = hotstuff.Committee{} - s.committee.On("Identities", mock.Anything).Return(nil, exception) +// Test_UnknownEpoch tests handling of a block from an unknown epoch. +// It should propagate the sentinel error model.ErrViewForUnknownEpoch from Committee. +func (s *blockSignerDecoderSuite) Test_UnknownEpoch() { + *s.committee = *hotstuff.NewDynamicCommittee(s.T()) + s.committee.On("IdentitiesByEpoch", mock.Anything).Return(nil, model.ErrViewForUnknownEpoch) ids, err := s.decoder.DecodeSignerIDs(s.block.Header) require.Empty(s.T(), ids) - require.False(s.T(), state.IsUnknownBlockError(err)) - require.True(s.T(), errors.Is(err, exception)) + require.ErrorIs(s.T(), err, model.ErrViewForUnknownEpoch) } // Test_InvalidIndices verifies that `BlockSignerDecoder` returns -// signature.InvalidSignerIndicesError is the signer indices in the provided header +// signature.InvalidSignerIndicesError if the signer indices in the provided header // are not a valid encoding. func (s *blockSignerDecoderSuite) Test_InvalidIndices() { s.block.Header.ParentVoterIndices = unittest.RandomBytes(1) @@ -100,3 +99,26 @@ func (s *blockSignerDecoderSuite) Test_InvalidIndices() { require.Empty(s.T(), ids) require.True(s.T(), signature.IsInvalidSignerIndicesError(err)) } + +// Test_EpochTransition verifies that `BlockSignerDecoder` correctly handles blocks +// near a boundary where the committee changes - an epoch transition. +func (s *blockSignerDecoderSuite) Test_EpochTransition() { + // The block under test B is the first block of a new epoch, where the committee changed. + // B contains a QC formed during the view of B's parent -- hence B's signatures must + // be decoded w.r.t. the committee as of the parent's view. + // + // Epoch 1 Epoch 2 + // PARENT <- | -- B + blockView := s.block.Header.View + parentView := s.block.Header.ParentView + epoch1Committee := s.allConsensus + epoch2Committee := s.allConsensus.SamplePct(.8) + + *s.committee = *hotstuff.NewDynamicCommittee(s.T()) + s.committee.On("IdentitiesByEpoch", parentView).Return(epoch1Committee, nil).Maybe() + s.committee.On("IdentitiesByEpoch", blockView).Return(epoch2Committee, nil).Maybe() + + ids, err := s.decoder.DecodeSignerIDs(s.block.Header) + require.NoError(s.T(), err) + require.Equal(s.T(), epoch1Committee.NodeIDs(), ids) +} diff --git a/consensus/hotstuff/signature/packer.go b/consensus/hotstuff/signature/packer.go index 15c43d5e2f2..f8a480fe5a0 100644 --- a/consensus/hotstuff/signature/packer.go +++ b/consensus/hotstuff/signature/packer.go @@ -1,6 +1,7 @@ package signature import ( + "errors" "fmt" "github.com/onflow/flow-go/consensus/hotstuff" @@ -13,13 +14,13 @@ import ( // The encoding method is RLP. type ConsensusSigDataPacker struct { model.SigDataPacker - committees hotstuff.Committee + committees hotstuff.Replicas } var _ hotstuff.Packer = &ConsensusSigDataPacker{} // NewConsensusSigDataPacker creates a new ConsensusSigDataPacker instance -func NewConsensusSigDataPacker(committees hotstuff.Committee) *ConsensusSigDataPacker { +func NewConsensusSigDataPacker(committees hotstuff.Replicas) *ConsensusSigDataPacker { return &ConsensusSigDataPacker{ committees: committees, } @@ -29,11 +30,11 @@ func NewConsensusSigDataPacker(committees hotstuff.Committee) *ConsensusSigDataP // To pack the block signature data, we first build a compact data type, and then encode it into bytes. // Expected error returns during normal operations: // - none; all errors are symptoms of inconsistent input data or corrupted internal state. -func (p *ConsensusSigDataPacker) Pack(blockID flow.Identifier, sig *hotstuff.BlockSignatureData) ([]byte, []byte, error) { +func (p *ConsensusSigDataPacker) Pack(view uint64, sig *hotstuff.BlockSignatureData) ([]byte, []byte, error) { // retrieve all authorized consensus participants at the given block - fullMembers, err := p.committees.Identities(blockID) + fullMembers, err := p.committees.IdentitiesByEpoch(view) if err != nil { - return nil, nil, fmt.Errorf("could not find consensus committees by block id(%v): %w", blockID, err) + return nil, nil, fmt.Errorf("could not find consensus committee for view %d: %w", view, err) } // breaking staking and random beacon signers into signerIDs and sig type for compaction @@ -63,6 +64,7 @@ func (p *ConsensusSigDataPacker) Pack(blockID flow.Identifier, sig *hotstuff.Blo } // Unpack de-serializes the provided signature data. +// view is the view of the block that the aggregated sig is signed for // sig is the aggregated signature data // It returns: // - (sigData, nil) if successfully unpacked the signature data @@ -76,10 +78,10 @@ func (p *ConsensusSigDataPacker) Unpack(signerIdentities flow.IdentityList, sigD stakingSigners, randomBeaconSigners, err := signature.DecodeSigTypeToStakingAndBeaconSigners(signerIdentities, data.SigType) if err != nil { - if signature.IsInvalidSigTypesError(err) { - return nil, model.NewInvalidFormatErrorf("invalid signer type data.SigType %v: %w", data.SigType, err) + if errors.Is(err, signature.ErrIllegallyPaddedBitVector) || errors.Is(err, signature.ErrIncompatibleBitVectorLength) { + return nil, model.NewInvalidFormatErrorf("invalid SigType vector: %w", err) } - return nil, fmt.Errorf("unexpected exception unpacking signer data data.SigType %v: %w", data.SigType, err) + return nil, fmt.Errorf("could not decode signer indices and sig type: %w", err) } return &hotstuff.BlockSignatureData{ diff --git a/consensus/hotstuff/signature/packer_test.go b/consensus/hotstuff/signature/packer_test.go index 67baa4326f5..fa5799c3119 100644 --- a/consensus/hotstuff/signature/packer_test.go +++ b/consensus/hotstuff/signature/packer_test.go @@ -2,6 +2,7 @@ package signature import ( "fmt" + "math/rand" "testing" "github.com/stretchr/testify/mock" @@ -17,9 +18,9 @@ import ( func newPacker(identities flow.IdentityList) *ConsensusSigDataPacker { // mock consensus committee - committee := &mocks.Committee{} - committee.On("Identities", mock.Anything).Return( - func(blockID flow.Identifier) flow.IdentityList { + committee := &mocks.DynamicCommittee{} + committee.On("IdentitiesByEpoch", mock.Anything).Return( + func(_ uint64) flow.IdentityList { return identities }, nil, @@ -54,14 +55,14 @@ func makeBlockSigData(committee flow.IdentityList) *hotstuff.BlockSignatureData func TestPackUnpack(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(6, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := makeBlockSigData(committee) // create packer with the committee packer := newPacker(committee) // pack & unpack - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -89,7 +90,7 @@ func TestPackUnpack(t *testing.T) { func TestPackUnpackManyNodes(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(200, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := makeBlockSigData(committee) stakingSigners := make([]flow.Identifier, 0) for i := 0; i < 60; i++ { @@ -106,7 +107,7 @@ func TestPackUnpackManyNodes(t *testing.T) { packer := newPacker(committee) // pack & unpack - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -133,13 +134,13 @@ func TestPackUnpackManyNodes(t *testing.T) { func TestFailToDecode(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(6, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := makeBlockSigData(committee) // create packer with the committee packer := newPacker(committee) - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -156,13 +157,13 @@ func TestFailToDecode(t *testing.T) { func TestMismatchSignerIDs(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(9, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := makeBlockSigData(committee[:6]) // create packer with the committee packer := newPacker(committee) - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -188,13 +189,13 @@ func TestMismatchSignerIDs(t *testing.T) { func TestInvalidSigType(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(6, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := makeBlockSigData(committee) // create packer with the committee packer := newPacker(committee) - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -222,7 +223,7 @@ func TestInvalidSigType(t *testing.T) { func TestPackUnpackWithoutRBAggregatedSig(t *testing.T) { // prepare data for testing committee := unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus)) - blockID := unittest.IdentifierFixture() + view := rand.Uint64() blockSigData := &hotstuff.BlockSignatureData{ StakingSigners: committee.NodeIDs(), @@ -236,7 +237,7 @@ func TestPackUnpackWithoutRBAggregatedSig(t *testing.T) { packer := newPacker(committee) // pack & unpack - signerIndices, sig, err := packer.Pack(blockID, blockSigData) + signerIndices, sig, err := packer.Pack(view, blockSigData) require.NoError(t, err) signers, err := signature.DecodeSignerIndicesToIdentities(committee, signerIndices) @@ -267,7 +268,7 @@ func TestPackWithoutRBAggregatedSig(t *testing.T) { committee := identities.NodeIDs() // prepare data for testing - blockID := unittest.IdentifierFixture() + view := rand.Uint64() aggregatedSig := unittest.SignatureFixture() reconstructedSig := unittest.SignatureFixture() @@ -292,10 +293,10 @@ func TestPackWithoutRBAggregatedSig(t *testing.T) { packer := newPacker(identities) // pack - signerIDs_A, sig_A, err := packer.Pack(blockID, blockSigDataWithEmptySlices) + signerIDs_A, sig_A, err := packer.Pack(view, blockSigDataWithEmptySlices) require.NoError(t, err) - signerIDs_B, sig_B, err := packer.Pack(blockID, blockSigDataWithNils) + signerIDs_B, sig_B, err := packer.Pack(view, blockSigDataWithNils) require.NoError(t, err) // should be the same diff --git a/consensus/hotstuff/signer.go b/consensus/hotstuff/signer.go index 50a4937c5e5..48d74c2bee4 100644 --- a/consensus/hotstuff/signer.go +++ b/consensus/hotstuff/signer.go @@ -2,6 +2,7 @@ package hotstuff import ( "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" ) // Signer is responsible for creating votes, proposals for a given block. @@ -13,4 +14,8 @@ type Signer interface { // CreateVote creates a vote for the given block. No error returns are // expected during normal operations (incl. presence of byz. actors). CreateVote(block *model.Block) (*model.Vote, error) + + // CreateTimeout creates a timeout for given view. No errors return are + // expected during normal operations(incl presence of byz. actors). + CreateTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) } diff --git a/consensus/hotstuff/timeout_aggregator.go b/consensus/hotstuff/timeout_aggregator.go new file mode 100644 index 00000000000..9dc86b587ea --- /dev/null +++ b/consensus/hotstuff/timeout_aggregator.go @@ -0,0 +1,26 @@ +package hotstuff + +import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/module" +) + +// TimeoutAggregator verifies and aggregates timeout objects to build timeout certificates [TCs]. +// When enough timeout objects are collected, it builds a TC and sends it to the EventLoop +// TimeoutAggregator also detects protocol violation, including invalid timeouts, double timeout, etc and +// notifies a HotStuff consumer for slashing. +type TimeoutAggregator interface { + module.ReadyDoneAware + module.Startable + + // AddTimeout verifies and aggregates a timeout object. + // This method can be called concurrently, timeouts will be queued and processed asynchronously. + AddTimeout(timeoutObject *model.TimeoutObject) + + // PruneUpToView deletes all `TimeoutCollector`s _below_ to the given view, as well as + // related indices. We only retain and process `TimeoutCollector`s, whose view is equal or larger + // than `lowestRetainedView`. If `lowestRetainedView` is smaller than the + // previous value, the previous value is kept and the method call is a NoOp. + // This value should be set to the latest active view maintained by `Pacemaker`. + PruneUpToView(lowestRetainedView uint64) +} diff --git a/consensus/hotstuff/timeout_collector.go b/consensus/hotstuff/timeout_collector.go new file mode 100644 index 00000000000..33898d7d37f --- /dev/null +++ b/consensus/hotstuff/timeout_collector.go @@ -0,0 +1,59 @@ +package hotstuff + +import ( + "github.com/onflow/flow-go/consensus/hotstuff/model" +) + +// TimeoutCollector collects all timeout objects for a specified view. On the happy path, it +// generates a TimeoutCertificate when enough timeouts have been collected. The TimeoutCollector +// is a higher-level structure that orchestrates deduplication, caching and processing of +// timeouts, delegating those tasks to underlying modules (such as TimeoutProcessor). +// Implementations of TimeoutCollector must be concurrency safe. +type TimeoutCollector interface { + // AddTimeout adds a Timeout Object [TO] to the collector. + // When TOs from strictly more than 1/3 of consensus participants (measured by weight) + // were collected, the callback for partial TC will be triggered. + // After collecting TOs from a supermajority, a TC will be created and passed to the EventLoop. + // Expected error returns during normal operations: + // * timeoutcollector.ErrTimeoutForIncompatibleView - submitted timeout for incompatible view + // All other exceptions are symptoms of potential state corruption. + AddTimeout(timeoutObject *model.TimeoutObject) error + + // View returns the view that this instance is collecting timeouts for. + // This method is useful when adding the newly created timeout collector to timeout collectors map. + View() uint64 +} + +// TimeoutProcessor ingests Timeout Objects [TO] for a particular view. It implements the algorithms +// for validating TOs, orchestrates their low-level aggregation and emits `OnPartialTcCreated` and +// `OnTcConstructedFromTimeouts` notifications. TimeoutProcessor cannot deduplicate TOs (this should +// be handled by the higher-level TimeoutCollector) and errors instead. +// Depending on their implementation, a TimeoutProcessor might drop timeouts or attempt to construct a TC. +type TimeoutProcessor interface { + // Process performs processing of single timeout object. This function is safe to call from multiple goroutines. + // Expected error returns during normal operations: + // * timeoutcollector.ErrTimeoutForIncompatibleView - submitted timeout for incompatible view + // * model.InvalidTimeoutError - submitted invalid timeout(invalid structure or invalid signature) + // * model.DuplicatedSignerError if a timeout from the same signer was previously already added + // It does _not necessarily_ imply that the timeout is invalid or the sender is equivocating. + // All other errors should be treated as exceptions. + Process(timeout *model.TimeoutObject) error +} + +// TimeoutCollectorFactory performs creation of TimeoutCollector for a given view +type TimeoutCollectorFactory interface { + // Create is a factory method to generate a TimeoutCollector for a given view + // Expected error returns during normal operations: + // * model.ErrViewForUnknownEpoch no epoch containing the given view is known + // All other errors should be treated as exceptions. + Create(view uint64) (TimeoutCollector, error) +} + +// TimeoutProcessorFactory performs creation of TimeoutProcessor for a given view +type TimeoutProcessorFactory interface { + // Create is a factory method to generate a TimeoutProcessor for a given view + // Expected error returns during normal operations: + // * model.ErrViewForUnknownEpoch no epoch containing the given view is known + // All other errors should be treated as exceptions. + Create(view uint64) (TimeoutProcessor, error) +} diff --git a/consensus/hotstuff/timeout_collectors.go b/consensus/hotstuff/timeout_collectors.go new file mode 100644 index 00000000000..0d0b23b63d4 --- /dev/null +++ b/consensus/hotstuff/timeout_collectors.go @@ -0,0 +1,26 @@ +package hotstuff + +// TimeoutCollectors encapsulates the functionality to generate, store and prune `TimeoutCollector` +// instances (one per view). Its main purpose is to provide a higher-level API to `TimeoutAggregator` +// for managing and interacting with the view-specific `TimeoutCollector` instances. +// Implementations are concurrency safe. +type TimeoutCollectors interface { + // GetOrCreateCollector retrieves the TimeoutCollector for the specified + // view or creates one if none exists. When creating a timeout collector, + // the view is used to query the consensus committee for the respective + // Epoch the view belongs to. + // It returns: + // - (collector, true, nil) if no collector can be found by the view, and a new collector was created. + // - (collector, false, nil) if the collector can be found by the view. + // - (nil, false, error) if running into any exception creating the timeout collector. + // Expected error returns during normal operations: + // * mempool.BelowPrunedThresholdError if view is below the pruning threshold + // * model.ErrViewForUnknownEpoch if view is not yet pruned but no epoch containing the given view is known + GetOrCreateCollector(view uint64) (collector TimeoutCollector, created bool, err error) + + // PruneUpToView prunes the timeout collectors with views _below_ the given value, i.e. + // we only retain and process timeout collectors, whose views are equal or larger than `lowestRetainedView`. + // If `lowestRetainedView` is smaller than the previous value, the previous value is + // kept and the method call is a NoOp. + PruneUpToView(lowestRetainedView uint64) +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go new file mode 100644 index 00000000000..ae308c42048 --- /dev/null +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator.go @@ -0,0 +1,237 @@ +package timeoutaggregator + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/common/fifoqueue" + "github.com/onflow/flow-go/engine/consensus/sealing/counters" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/metrics" +) + +// defaultTimeoutAggregatorWorkers number of workers to dispatch events for timeout aggregator +const defaultTimeoutAggregatorWorkers = 4 + +// defaultTimeoutQueueCapacity maximum capacity for buffering unprocessed timeouts +const defaultTimeoutQueueCapacity = 1000 + +// TimeoutAggregator stores the timeout objects and aggregates them into a TC when enough TOs have been collected. +// It's safe to use in concurrent environment. +type TimeoutAggregator struct { + *component.ComponentManager + notifications.NoopConsumer + log zerolog.Logger + hotstuffMetrics module.HotstuffMetrics + engineMetrics module.EngineMetrics + notifier hotstuff.Consumer + lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process timeouts + collectors hotstuff.TimeoutCollectors + queuedTimeoutsNotifier engine.Notifier + enteringViewNotifier engine.Notifier + queuedTimeouts *fifoqueue.FifoQueue +} + +var _ hotstuff.TimeoutAggregator = (*TimeoutAggregator)(nil) +var _ component.Component = (*TimeoutAggregator)(nil) + +// NewTimeoutAggregator creates an instance of timeout aggregator. +// No errors are expected during normal operations. +func NewTimeoutAggregator(log zerolog.Logger, + hotstuffMetrics module.HotstuffMetrics, + engineMetrics module.EngineMetrics, + mempoolMetrics module.MempoolMetrics, + notifier hotstuff.Consumer, + lowestRetainedView uint64, + collectors hotstuff.TimeoutCollectors, +) (*TimeoutAggregator, error) { + queuedTimeouts, err := fifoqueue.NewFifoQueue(defaultTimeoutQueueCapacity, + fifoqueue.WithLengthObserver(func(len int) { mempoolMetrics.MempoolEntries(metrics.ResourceTimeoutObjectQueue, uint(len)) })) + if err != nil { + return nil, fmt.Errorf("could not initialize timeouts queue") + } + + aggregator := &TimeoutAggregator{ + log: log.With().Str("component", "hotstuff.timeout_aggregator").Logger(), + hotstuffMetrics: hotstuffMetrics, + engineMetrics: engineMetrics, + notifier: notifier, + lowestRetainedView: counters.NewMonotonousCounter(lowestRetainedView), + collectors: collectors, + queuedTimeoutsNotifier: engine.NewNotifier(), + enteringViewNotifier: engine.NewNotifier(), + queuedTimeouts: queuedTimeouts, + } + + componentBuilder := component.NewComponentManagerBuilder() + for i := 0; i < defaultTimeoutAggregatorWorkers; i++ { // manager for worker routines that process inbound events + componentBuilder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + aggregator.queuedTimeoutsProcessingLoop(ctx) + }) + } + componentBuilder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + aggregator.enteringViewProcessingLoop(ctx) + }) + + aggregator.ComponentManager = componentBuilder.Build() + return aggregator, nil +} + +// queuedTimeoutsProcessingLoop is the event loop which waits for notification about pending work +// and as soon as there is some it triggers processing. +// All errors are propagated to SignalerContext +func (t *TimeoutAggregator) queuedTimeoutsProcessingLoop(ctx irrecoverable.SignalerContext) { + notifier := t.queuedTimeoutsNotifier.Channel() + for { + select { + case <-ctx.Done(): + return + case <-notifier: + err := t.processQueuedTimeoutObjects(ctx) + if err != nil { + ctx.Throw(fmt.Errorf("internal error processing queued timeout events: %w", err)) + return + } + } + } +} + +// processQueuedTimeoutObjects sequentially processes items from `queuedTimeouts` +// until the queue returns 'empty'. Only when there are no more queued up TimeoutObjects, +// this function call returns. +// No errors are expected during normal operations. +func (t *TimeoutAggregator) processQueuedTimeoutObjects(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + default: + } + + msg, ok := t.queuedTimeouts.Pop() + if !ok { + // when there is no more messages in the queue, back to the loop to wait + // for the next incoming message to arrive. + return nil + } + + timeoutObject := msg.(*model.TimeoutObject) + startTime := time.Now() + + err := t.processQueuedTimeout(timeoutObject) + // report duration of processing one timeout object + t.hotstuffMetrics.TimeoutObjectProcessingDuration(time.Since(startTime)) + t.engineMetrics.MessageHandled(metrics.EngineTimeoutAggregator, metrics.MessageTimeoutObject) + + if err != nil { + return fmt.Errorf("could not process pending TO %v: %w", timeoutObject.ID(), err) + } + + t.log.Info(). + Uint64("view", timeoutObject.View). + Hex("signer", timeoutObject.SignerID[:]). + Msg("TimeoutObject processed successfully") + } +} + +// processQueuedTimeout performs actual processing of queued timeouts, this method is called from multiple +// concurrent goroutines. +// No errors are expected during normal operation +func (t *TimeoutAggregator) processQueuedTimeout(timeoutObject *model.TimeoutObject) error { + // We create a timeout collector before validating the first TO, so processing an invalid TO will + // result in a collector being added, until the corresponding view is pruned. + // Since epochs on mainnet have 500,000-1,000,000 views, there is an opportunity for memory exhaustion here. + // However, because only invalid TOs can result in creating collectors at an arbitrary view, + // and invalid TOs are slashable, resulting eventually in ejection of the sender, this attack is impractical. + collector, _, err := t.collectors.GetOrCreateCollector(timeoutObject.View) + if err != nil { + // ignore if our routine is outdated and some other one has pruned collectors + if mempool.IsBelowPrunedThresholdError(err) { + return nil + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // ignore TO if we don't have information for epoch + t.log.Debug().Uint64("view", timeoutObject.View).Msg("discarding TO for view beyond known epochs") + return nil + } + return fmt.Errorf("could not get collector for view %d: %w", + timeoutObject.View, err) + } + + err = collector.AddTimeout(timeoutObject) + if err != nil { + return fmt.Errorf("could not process TO for view %d: %w", + timeoutObject.View, err) + } + return nil +} + +// AddTimeout checks if TO is stale and appends TO to processing queue. +// The actual processing will be done asynchronously by the `TimeoutAggregator`'s internal worker routines. +func (t *TimeoutAggregator) AddTimeout(timeoutObject *model.TimeoutObject) { + // drop stale objects + if timeoutObject.View < t.lowestRetainedView.Value() { + t.log.Debug(). + Uint64("view", timeoutObject.View). + Hex("signer", timeoutObject.SignerID[:]). + Msg("drop stale timeouts") + t.engineMetrics.InboundMessageDropped(metrics.EngineTimeoutAggregator, metrics.MessageTimeoutObject) + return + } + + placedInQueue := t.queuedTimeouts.Push(timeoutObject) + if !placedInQueue { // processing pipeline `queuedTimeouts` is full + // It's ok to silently drop timeouts, because we are probably catching up. + t.log.Info(). + Uint64("view", timeoutObject.View). + Hex("signer", timeoutObject.SignerID[:]). + Msg("no queue capacity, dropping timeout") + t.engineMetrics.InboundMessageDropped(metrics.EngineTimeoutAggregator, metrics.MessageTimeoutObject) + return + } + t.queuedTimeoutsNotifier.Notify() +} + +// PruneUpToView deletes all `TimeoutCollector`s _below_ to the given view, as well as +// related indices. We only retain and process `TimeoutCollector`s, whose view is equal or larger +// than `lowestRetainedView`. If `lowestRetainedView` is smaller than the +// previous value, the previous value is kept and the method call is a NoOp. +func (t *TimeoutAggregator) PruneUpToView(lowestRetainedView uint64) { + t.collectors.PruneUpToView(lowestRetainedView) +} + +// OnViewChange implements the `OnViewChange` callback from the `hotstuff.Consumer`. +// We notify the enteringViewProcessingLoop worker, which then prunes up to the active view. +// CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages +// from external nodes cannot be considered as inputs to this function +func (t *TimeoutAggregator) OnViewChange(_, newView uint64) { + if t.lowestRetainedView.Set(newView) { + t.enteringViewNotifier.Notify() + } +} + +// enteringViewProcessingLoop is a separate goroutine that performs processing of entering view events +func (t *TimeoutAggregator) enteringViewProcessingLoop(ctx context.Context) { + notifier := t.enteringViewNotifier.Channel() + for { + select { + case <-ctx.Done(): + return + case <-notifier: + t.PruneUpToView(t.lowestRetainedView.Value()) + } + } +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go new file mode 100644 index 00000000000..fddce6bd717 --- /dev/null +++ b/consensus/hotstuff/timeoutaggregator/timeout_aggregator_test.go @@ -0,0 +1,146 @@ +package timeoutaggregator + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestTimeoutAggregator(t *testing.T) { + suite.Run(t, new(TimeoutAggregatorTestSuite)) +} + +// TimeoutAggregatorTestSuite is a test suite for isolated testing of TimeoutAggregator. +// Contains mocked state which is used to verify correct behavior of TimeoutAggregator. +// Automatically starts and stops module.Startable in SetupTest and TearDownTest respectively. +type TimeoutAggregatorTestSuite struct { + suite.Suite + + lowestRetainedView uint64 + highestKnownView uint64 + aggregator *TimeoutAggregator + collectors *mocks.TimeoutCollectors + consumer *mocks.Consumer + stopAggregator context.CancelFunc +} + +func (s *TimeoutAggregatorTestSuite) SetupTest() { + var err error + s.collectors = mocks.NewTimeoutCollectors(s.T()) + s.consumer = mocks.NewConsumer(s.T()) + + s.lowestRetainedView = 100 + + metricsCollector := metrics.NewNoopCollector() + + s.aggregator, err = NewTimeoutAggregator( + unittest.Logger(), + metricsCollector, + metricsCollector, + metricsCollector, + s.consumer, + s.lowestRetainedView, + s.collectors, + ) + require.NoError(s.T(), err) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx, _ := irrecoverable.WithSignaler(ctx) + s.stopAggregator = cancel + s.aggregator.Start(signalerCtx) + unittest.RequireCloseBefore(s.T(), s.aggregator.Ready(), 100*time.Millisecond, "should close before timeout") +} + +func (s *TimeoutAggregatorTestSuite) TearDownTest() { + s.stopAggregator() + unittest.RequireCloseBefore(s.T(), s.aggregator.Done(), time.Second, "should close before timeout") +} + +// TestAddTimeout_HappyPath tests a happy path when multiple threads are adding timeouts for processing +// Eventually every timeout has to be processed by TimeoutCollector +func (s *TimeoutAggregatorTestSuite) TestAddTimeout_HappyPath() { + timeoutsCount := 20 + collector := mocks.NewTimeoutCollector(s.T()) + callCount := atomic.NewUint64(0) + collector.On("AddTimeout", mock.Anything).Run(func(mock.Arguments) { + callCount.Add(1) + }).Return(nil).Times(timeoutsCount) + s.collectors.On("GetOrCreateCollector", s.lowestRetainedView).Return(collector, true, nil).Times(timeoutsCount) + + var start sync.WaitGroup + start.Add(timeoutsCount) + for i := 0; i < timeoutsCount; i++ { + go func() { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.lowestRetainedView)) + + start.Done() + // Wait for last worker routine to signal ready. Then, + // feed all timeouts into cache + start.Wait() + + s.aggregator.AddTimeout(timeout) + }() + } + + start.Wait() + + require.Eventually(s.T(), func() bool { + return callCount.Load() == uint64(timeoutsCount) + }, time.Second, time.Millisecond*20) +} + +// TestAddTimeout_EpochUnknown tests if timeout objects targeting unknown epoch should be ignored +func (s *TimeoutAggregatorTestSuite) TestAddTimeout_EpochUnknown() { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.lowestRetainedView)) + *s.collectors = *mocks.NewTimeoutCollectors(s.T()) + done := make(chan struct{}) + s.collectors.On("GetOrCreateCollector", timeout.View).Return(nil, false, model.ErrViewForUnknownEpoch).Run(func(args mock.Arguments) { + close(done) + }).Once() + s.aggregator.AddTimeout(timeout) + unittest.AssertClosesBefore(s.T(), done, time.Second) +} + +// TestPruneUpToView tests that pruning removes collectors lower that retained view +func (s *TimeoutAggregatorTestSuite) TestPruneUpToView() { + s.collectors.On("PruneUpToView", s.lowestRetainedView+1).Once() + s.aggregator.PruneUpToView(s.lowestRetainedView + 1) +} + +// TestOnQcTriggeredViewChange tests if entering view event gets processed when send through `TimeoutAggregator`. +// Tests the whole processing pipeline. +func (s *TimeoutAggregatorTestSuite) TestOnQcTriggeredViewChange() { + done := make(chan struct{}) + s.collectors.On("PruneUpToView", s.lowestRetainedView+1).Run(func(args mock.Arguments) { + close(done) + }).Once() + qc := helper.MakeQC(helper.WithQCView(s.lowestRetainedView)) + s.aggregator.OnViewChange(qc.View, qc.View+1) + unittest.AssertClosesBefore(s.T(), done, time.Second) +} + +// TestOnTcTriggeredViewChange tests if entering view event gets processed when send through `TimeoutAggregator`. +// Tests the whole processing pipeline. +func (s *TimeoutAggregatorTestSuite) TestOnTcTriggeredViewChange() { + view := s.lowestRetainedView + 1 + done := make(chan struct{}) + s.collectors.On("PruneUpToView", view).Run(func(args mock.Arguments) { + close(done) + }).Once() + tc := helper.MakeTC(helper.WithTCView(s.lowestRetainedView)) + s.aggregator.OnViewChange(tc.View, tc.View+1) + unittest.AssertClosesBefore(s.T(), done, time.Second) +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_collectors.go b/consensus/hotstuff/timeoutaggregator/timeout_collectors.go new file mode 100644 index 00000000000..20369bc9485 --- /dev/null +++ b/consensus/hotstuff/timeoutaggregator/timeout_collectors.go @@ -0,0 +1,134 @@ +package timeoutaggregator + +import ( + "fmt" + "sync" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/module/mempool" +) + +// TimeoutCollectors implements management of multiple timeout collectors indexed by view. +// Implements hotstuff.TimeoutCollectors interface. Creating a TimeoutCollector for a +// particular view is lazy (instances are created on demand). +// This structure is concurrently safe. +// TODO: once VoteCollectors gets updated to stop managing worker pool we can merge VoteCollectors and TimeoutCollectors using generics +// TODO(active-pacemaker): add metrics for tracking size of collectors and active range +type TimeoutCollectors struct { + log zerolog.Logger + lock sync.RWMutex + lowestRetainedView uint64 // lowest view, for which we still retain a TimeoutCollector and process timeouts + collectors map[uint64]hotstuff.TimeoutCollector // view -> TimeoutCollector + collectorFactory hotstuff.TimeoutCollectorFactory // factor for creating collectors +} + +var _ hotstuff.TimeoutCollectors = (*TimeoutCollectors)(nil) + +func NewTimeoutCollectors(log zerolog.Logger, lowestRetainedView uint64, collectorFactory hotstuff.TimeoutCollectorFactory) *TimeoutCollectors { + return &TimeoutCollectors{ + log: log.With().Str("component", "timeout_collectors").Logger(), + lowestRetainedView: lowestRetainedView, + collectors: make(map[uint64]hotstuff.TimeoutCollector), + collectorFactory: collectorFactory, + } +} + +// GetOrCreateCollector retrieves the hotstuff.TimeoutCollector for the specified +// view or creates one if none exists. +// - (collector, true, nil) if no collector can be found by the view, and a new collector was created. +// - (collector, false, nil) if the collector can be found by the view +// - (nil, false, error) if running into any exception creating the timeout collector state machine +// +// Expected error returns during normal operations: +// - mempool.BelowPrunedThresholdError if view is below the pruning threshold +// - model.ErrViewForUnknownEpoch if view is not yet pruned but no epoch containing the given view is known, this error +// +// can be returned from factory method. +func (t *TimeoutCollectors) GetOrCreateCollector(view uint64) (hotstuff.TimeoutCollector, bool, error) { + cachedCollector, hasCachedCollector, err := t.getCollector(view) + if err != nil { + return nil, false, err + } + if hasCachedCollector { + return cachedCollector, false, nil + } + + collector, err := t.collectorFactory.Create(view) + if err != nil { + return nil, false, fmt.Errorf("could not create timeout collector for view %d: %w", view, err) + } + + // Initial check showed that there was no collector. However, it's possible that after the + // initial check but before acquiring the lock to add the newly-created collector, another + // goroutine already added the needed collector. Hence, check again after acquiring the lock: + t.lock.Lock() + clr, found := t.collectors[view] + if found { + t.lock.Unlock() + return clr, false, nil + } + t.collectors[view] = collector + t.lock.Unlock() + + t.log.Info().Uint64("view", view).Msg("timeout collector has been created") + return collector, true, nil +} + +// getCollector retrieves hotstuff.TimeoutCollector from local cache in concurrent safe way. +// Performs check for lowestRetainedView. +// Expected error returns during normal operations: +// - mempool.BelowPrunedThresholdError - in case view is lower than lowestRetainedView +func (t *TimeoutCollectors) getCollector(view uint64) (hotstuff.TimeoutCollector, bool, error) { + t.lock.RLock() + defer t.lock.RUnlock() + if view < t.lowestRetainedView { + return nil, false, mempool.NewBelowPrunedThresholdErrorf("cannot retrieve collector for pruned view %d (lowest retained view %d)", view, t.lowestRetainedView) + } + + clr, found := t.collectors[view] + return clr, found, nil +} + +// PruneUpToView prunes the timeout collectors with views _below_ the given value, i.e. +// we only retain and process whose view is equal or larger than `lowestRetainedView`. +// If `lowestRetainedView` is smaller than the previous value, the previous value is +// kept and the method call is a NoOp. +func (t *TimeoutCollectors) PruneUpToView(lowestRetainedView uint64) { + t.lock.Lock() + defer t.lock.Unlock() + if t.lowestRetainedView >= lowestRetainedView { + return + } + sizeBefore := len(t.collectors) + if sizeBefore == 0 { + t.lowestRetainedView = lowestRetainedView + return + } + + // to optimize the pruning of large view-ranges, we compare: + // * the number of views for which we have collectors: len(t.collectors) + // * the number of views that need to be pruned: view-t.lowestRetainedView + // We iterate over the dimension which is smaller. + if uint64(sizeBefore) < lowestRetainedView-t.lowestRetainedView { + for w := range t.collectors { + if w < lowestRetainedView { + delete(t.collectors, w) + } + } + } else { + for w := t.lowestRetainedView; w < lowestRetainedView; w++ { + delete(t.collectors, w) + } + } + from := t.lowestRetainedView + t.lowestRetainedView = lowestRetainedView + + t.log.Debug(). + Uint64("prior_lowest_retained_view", from). + Uint64("lowest_retained_view", lowestRetainedView). + Int("prior_size", sizeBefore). + Int("size", len(t.collectors)). + Msgf("pruned timeout collectors") +} diff --git a/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go b/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go new file mode 100644 index 00000000000..66252c6e065 --- /dev/null +++ b/consensus/hotstuff/timeoutaggregator/timeout_collectors_test.go @@ -0,0 +1,178 @@ +package timeoutaggregator + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/gammazero/workerpool" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/utils/unittest" +) + +var factoryError = errors.New("factory error") + +func TestTimeoutCollectors(t *testing.T) { + suite.Run(t, new(TimeoutCollectorsTestSuite)) +} + +// TimeoutCollectorsTestSuite is a test suite for isolated testing of TimeoutCollectors. +// Contains helper methods and mocked state which is used to verify correct behavior of TimeoutCollectors. +type TimeoutCollectorsTestSuite struct { + suite.Suite + + mockedCollectors map[uint64]*mocks.TimeoutCollector + factoryMethod *mocks.TimeoutCollectorFactory + collectors *TimeoutCollectors + lowestView uint64 + workerPool *workerpool.WorkerPool +} + +func (s *TimeoutCollectorsTestSuite) SetupTest() { + s.lowestView = 1000 + s.mockedCollectors = make(map[uint64]*mocks.TimeoutCollector) + s.workerPool = workerpool.New(2) + s.factoryMethod = mocks.NewTimeoutCollectorFactory(s.T()) + s.factoryMethod.On("Create", mock.Anything).Return(func(view uint64) hotstuff.TimeoutCollector { + if collector, found := s.mockedCollectors[view]; found { + return collector + } + return nil + }, func(view uint64) error { + if _, found := s.mockedCollectors[view]; found { + return nil + } + return fmt.Errorf("mocked collector %v not found: %w", view, factoryError) + }).Maybe() + s.collectors = NewTimeoutCollectors(unittest.Logger(), s.lowestView, s.factoryMethod) +} + +func (s *TimeoutCollectorsTestSuite) TearDownTest() { + s.workerPool.StopWait() +} + +// prepareMockedCollector prepares a mocked collector and stores it in map, later it will be used +// to mock behavior of timeout collectors. +func (s *TimeoutCollectorsTestSuite) prepareMockedCollector(view uint64) *mocks.TimeoutCollector { + collector := mocks.NewTimeoutCollector(s.T()) + collector.On("View").Return(view).Maybe() + s.mockedCollectors[view] = collector + return collector +} + +// TestGetOrCreateCollector_ViewLowerThanLowest tests a scenario where caller tries to create a collector with view +// lower than already pruned one. This should result in sentinel error `BelowPrunedThresholdError` +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_ViewLowerThanLowest() { + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestView - 10) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.Error(s.T(), err) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) +} + +// TestGetOrCreateCollector_UnknownEpoch tests a scenario where caller tries to create a collector with view referring epoch +// that we don't know about. This should result in sentinel error ` +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_UnknownEpoch() { + *s.factoryMethod = *mocks.NewTimeoutCollectorFactory(s.T()) + s.factoryMethod.On("Create", mock.Anything).Return(nil, model.ErrViewForUnknownEpoch) + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestView + 100) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.ErrorIs(s.T(), err, model.ErrViewForUnknownEpoch) +} + +// TestGetOrCreateCollector_ValidCollector tests a happy path scenario where we try first to create and then retrieve cached collector. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_ValidCollector() { + view := s.lowestView + 10 + s.prepareMockedCollector(view) + collector, created, err := s.collectors.GetOrCreateCollector(view) + require.NoError(s.T(), err) + require.True(s.T(), created) + require.Equal(s.T(), view, collector.View()) + + cached, cachedCreated, err := s.collectors.GetOrCreateCollector(view) + require.NoError(s.T(), err) + require.False(s.T(), cachedCreated) + require.Equal(s.T(), collector, cached) +} + +// TestGetOrCreateCollector_FactoryError tests that error from factory method is propagated to caller. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_FactoryError() { + // creating collector without calling prepareMockedCollector will yield factoryError. + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestView + 10) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.ErrorIs(s.T(), err, factoryError) +} + +// TestGetOrCreateCollectors_ConcurrentAccess tests that concurrently accessing of GetOrCreateCollector creates +// only one collector and all other instances are retrieved from cache. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollectors_ConcurrentAccess() { + createdTimes := atomic.NewUint64(0) + view := s.lowestView + 10 + s.prepareMockedCollector(view) + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, created, err := s.collectors.GetOrCreateCollector(view) + require.NoError(s.T(), err) + if created { + createdTimes.Add(1) + } + }() + } + + unittest.AssertReturnsBefore(s.T(), wg.Wait, time.Second) + require.Equal(s.T(), uint64(1), createdTimes.Load()) +} + +// TestPruneUpToView tests pruning removes item below pruning height and leaves unmodified other items. +func (s *TimeoutCollectorsTestSuite) TestPruneUpToView() { + numberOfCollectors := uint64(10) + prunedViews := make([]uint64, 0) + for i := uint64(0); i < numberOfCollectors; i++ { + view := s.lowestView + i + s.prepareMockedCollector(view) + _, _, err := s.collectors.GetOrCreateCollector(view) + require.NoError(s.T(), err) + prunedViews = append(prunedViews, view) + } + + pruningHeight := s.lowestView + numberOfCollectors + + expectedCollectors := make([]hotstuff.TimeoutCollector, 0) + for i := uint64(0); i < numberOfCollectors; i++ { + view := pruningHeight + i + s.prepareMockedCollector(view) + collector, _, err := s.collectors.GetOrCreateCollector(view) + require.NoError(s.T(), err) + expectedCollectors = append(expectedCollectors, collector) + } + + // after this operation collectors below pruning height should be pruned and everything higher + // should be left unmodified + s.collectors.PruneUpToView(pruningHeight) + + for _, prunedView := range prunedViews { + _, _, err := s.collectors.GetOrCreateCollector(prunedView) + require.Error(s.T(), err) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) + } + + for _, collector := range expectedCollectors { + cached, _, _ := s.collectors.GetOrCreateCollector(collector.View()) + require.Equal(s.T(), collector, cached) + } +} diff --git a/consensus/hotstuff/timeoutcollector/aggregation.go b/consensus/hotstuff/timeoutcollector/aggregation.go new file mode 100644 index 00000000000..0fc8a4db575 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/aggregation.go @@ -0,0 +1,194 @@ +package timeoutcollector + +import ( + "fmt" + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/verification" + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/model/flow" + msig "github.com/onflow/flow-go/module/signature" +) + +// signerInfo holds information about a signer, its public key and weight +type signerInfo struct { + pk crypto.PublicKey + weight uint64 +} + +// sigInfo holds signature and high QC view submitted by some signer +type sigInfo struct { + sig crypto.Signature + newestQCView uint64 +} + +// TimeoutSignatureAggregator implements consensus/hotstuff.TimeoutSignatureAggregator. +// It performs timeout specific BLS aggregation over multiple distinct messages. +// We perform timeout signature aggregation for some concrete view, utilizing the protocol specification +// that timeouts sign the message: hash(view, newestQCView), where newestQCView can have different values +// for different replicas. +// View and the identities of all authorized replicas are +// specified when the TimeoutSignatureAggregator is instantiated. +// Each signer is allowed to sign at most once. +// Aggregation uses BLS scheme. Mitigation against rogue attacks is done using Proof Of Possession (PoP) +// This module does not verify PoPs of input public keys, it assumes verification was done outside this module. +// Implementation is thread-safe. +type TimeoutSignatureAggregator struct { + lock sync.RWMutex + hasher hash.Hasher + idToInfo map[flow.Identifier]signerInfo // auxiliary map to lookup signer weight and public key (only gets updated by constructor) + idToSignature map[flow.Identifier]sigInfo // signatures indexed by the signer ID + totalWeight uint64 // total accumulated weight + view uint64 // view for which we are aggregating signatures +} + +var _ hotstuff.TimeoutSignatureAggregator = (*TimeoutSignatureAggregator)(nil) + +// NewTimeoutSignatureAggregator returns a multi message signature aggregator initialized with a predefined view +// for which we aggregate signatures, list of flow identities, +// their respective public keys and a domain separation tag. The identities +// represent the list of all authorized signers. +// The constructor errors if: +// - the list of identities is empty +// - if one of the keys is not a valid public key. +// +// A multi message sig aggregator is used for aggregating timeouts for a single view only. A new instance should be used for each +// signature aggregation task in the protocol. +func NewTimeoutSignatureAggregator( + view uint64, // view for which we are aggregating signatures + ids flow.IdentityList, // list of all authorized signers + dsTag string, // domain separation tag used by the signature +) (*TimeoutSignatureAggregator, error) { + if len(ids) == 0 { + return nil, fmt.Errorf("number of participants must be larger than 0, got %d", len(ids)) + } + // sanity check for BLS keys + for i, identity := range ids { + if identity.StakingPubKey.Algorithm() != crypto.BLSBLS12381 { + return nil, fmt.Errorf("key at index %d is not a BLS key", i) + } + } + + // build the internal map for a faster look-up + idToInfo := make(map[flow.Identifier]signerInfo) + for _, id := range ids { + idToInfo[id.NodeID] = signerInfo{ + pk: id.StakingPubKey, + weight: id.Weight, + } + } + + return &TimeoutSignatureAggregator{ + hasher: msig.NewBLSHasher(dsTag), // concurrency safe + idToInfo: idToInfo, + idToSignature: make(map[flow.Identifier]sigInfo), + view: view, + }, nil +} + +// VerifyAndAdd verifies the signature under the stored public keys and adds signature with corresponding +// newest QC view to the internal set. Internal set and collected weight is modified iff the signer ID is not a duplicate and signature _is_ valid. +// The total weight of all collected signatures (excluding duplicates) is returned regardless +// of any returned error. +// Expected errors during normal operations: +// - model.InvalidSignerError if signerID is invalid (not a consensus participant) +// - model.DuplicatedSignerError if the signer has been already added +// - model.ErrInvalidSignature if signerID is valid but signature is cryptographically invalid +// +// The function is thread-safe. +func (a *TimeoutSignatureAggregator) VerifyAndAdd(signerID flow.Identifier, sig crypto.Signature, newestQCView uint64) (totalWeight uint64, exception error) { + info, ok := a.idToInfo[signerID] + if !ok { + return a.TotalWeight(), model.NewInvalidSignerErrorf("%v is not an authorized signer", signerID) + } + + // to avoid expensive signature verification we will proceed with double lock style check + if a.hasSignature(signerID) { + return a.TotalWeight(), model.NewDuplicatedSignerErrorf("signature from %v was already added", signerID) + } + + msg := verification.MakeTimeoutMessage(a.view, newestQCView) + valid, err := info.pk.Verify(sig, msg, a.hasher) + if err != nil { + return a.TotalWeight(), fmt.Errorf("couldn't verify signature from %s: %w", signerID, err) + } + if !valid { + return a.TotalWeight(), fmt.Errorf("invalid signature from %s: %w", signerID, model.ErrInvalidSignature) + } + + a.lock.Lock() + defer a.lock.Unlock() + + if _, duplicate := a.idToSignature[signerID]; duplicate { + return a.totalWeight, model.NewDuplicatedSignerErrorf("signature from %v was already added", signerID) + } + + a.idToSignature[signerID] = sigInfo{ + sig: sig, + newestQCView: newestQCView, + } + a.totalWeight += info.weight + + return a.totalWeight, nil +} + +func (a *TimeoutSignatureAggregator) hasSignature(singerID flow.Identifier) bool { + a.lock.RLock() + defer a.lock.RUnlock() + _, found := a.idToSignature[singerID] + return found +} + +// TotalWeight returns the total weight presented by the collected signatures. +// The function is thread-safe +func (a *TimeoutSignatureAggregator) TotalWeight() uint64 { + a.lock.RLock() + defer a.lock.RUnlock() + return a.totalWeight +} + +// View returns view for which aggregation happens +// The function is thread-safe +func (a *TimeoutSignatureAggregator) View() uint64 { + return a.view +} + +// Aggregate aggregates the signatures and returns the aggregated signature. +// The resulting aggregated signature is guaranteed to be valid, as all individual +// signatures are pre-validated before their addition. +// Expected errors during normal operations: +// - model.InsufficientSignaturesError if no signatures have been added yet +// +// This function is thread-safe +func (a *TimeoutSignatureAggregator) Aggregate() ([]hotstuff.TimeoutSignerInfo, crypto.Signature, error) { + a.lock.RLock() + defer a.lock.RUnlock() + + sharesNum := len(a.idToSignature) + if sharesNum == 0 { + return nil, nil, model.NewInsufficientSignaturesErrorf("cannot aggregate an empty list of signatures") + } + signatures := make([]crypto.Signature, 0, sharesNum) + signersData := make([]hotstuff.TimeoutSignerInfo, 0, sharesNum) + for id, info := range a.idToSignature { + signatures = append(signatures, info.sig) + signersData = append(signersData, hotstuff.TimeoutSignerInfo{ + NewestQCView: info.newestQCView, + Signer: id, + }) + } + + aggSignature, err := crypto.AggregateBLSSignatures(signatures) + if err != nil { + // unexpected error for: + // * empty `signatures` slice, i.e. sharesNum == 0, which we exclude by earlier check + // * if some signature(s) could not be decoded, which should be impossible since we check all signatures before adding them + // Hence, any error here is a symptom of an internal bug + return nil, nil, fmt.Errorf("unexpected internal error during BLS signature aggregation: %w", err) + } + + return signersData, aggSignature, nil +} diff --git a/consensus/hotstuff/timeoutcollector/aggregation_test.go b/consensus/hotstuff/timeoutcollector/aggregation_test.go new file mode 100644 index 00000000000..90bbef4632a --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/aggregation_test.go @@ -0,0 +1,241 @@ +package timeoutcollector + +import ( + "math/rand" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/verification" + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/model/flow" + msig "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/utils/unittest" +) + +// createAggregationData is a helper which creates fixture data for testing +func createAggregationData(t *testing.T, signersNumber int) ( + *TimeoutSignatureAggregator, + flow.IdentityList, + []crypto.PublicKey, + []crypto.Signature, + []hotstuff.TimeoutSignerInfo, + [][]byte, + []hash.Hasher) { + + // create message and tag + tag := "random_tag" + hasher := msig.NewBLSHasher(tag) + sigs := make([]crypto.Signature, 0, signersNumber) + signersInfo := make([]hotstuff.TimeoutSignerInfo, 0, signersNumber) + msgs := make([][]byte, 0, signersNumber) + hashers := make([]hash.Hasher, 0, signersNumber) + + // create keys, identities and signatures + ids := make([]*flow.Identity, 0, signersNumber) + pks := make([]crypto.PublicKey, 0, signersNumber) + view := 10 + uint64(rand.Uint32()) + for i := 0; i < signersNumber; i++ { + sk := unittest.PrivateKeyFixture(crypto.BLSBLS12381, crypto.KeyGenSeedMinLenECDSAP256) + identity := unittest.IdentityFixture(unittest.WithStakingPubKey(sk.PublicKey())) + // id + ids = append(ids, identity) + // keys + newestQCView := uint64(rand.Intn(int(view))) + msg := verification.MakeTimeoutMessage(view, newestQCView) + // signatures + sig, err := sk.Sign(msg, hasher) + require.NoError(t, err) + sigs = append(sigs, sig) + + pks = append(pks, identity.StakingPubKey) + signersInfo = append(signersInfo, hotstuff.TimeoutSignerInfo{ + NewestQCView: newestQCView, + Signer: identity.NodeID, + }) + hashers = append(hashers, hasher) + msgs = append(msgs, msg) + } + aggregator, err := NewTimeoutSignatureAggregator(view, ids, tag) + require.NoError(t, err) + return aggregator, ids, pks, sigs, signersInfo, msgs, hashers +} + +// TestNewTimeoutSignatureAggregator tests different happy and unhappy path scenarios when constructing +// multi message signature aggregator. +func TestNewTimeoutSignatureAggregator(t *testing.T) { + tag := "random_tag" + + sk := unittest.PrivateKeyFixture(crypto.ECDSAP256, crypto.KeyGenSeedMinLenECDSAP256) + signer := unittest.IdentityFixture(unittest.WithStakingPubKey(sk.PublicKey())) + // wrong key type + _, err := NewTimeoutSignatureAggregator(0, flow.IdentityList{signer}, tag) + require.Error(t, err) + // empty signers + _, err = NewTimeoutSignatureAggregator(0, flow.IdentityList{}, tag) + require.Error(t, err) +} + +// TestTimeoutSignatureAggregator_HappyPath tests happy path when aggregating signatures +// Tests verification, adding and aggregation. Test is performed in concurrent environment +func TestTimeoutSignatureAggregator_HappyPath(t *testing.T) { + signersNum := 20 + aggregator, ids, pks, sigs, signersData, msgs, hashers := createAggregationData(t, signersNum) + + // only add a subset of the signatures + subSet := signersNum / 2 + expectedWeight := uint64(0) + var wg sync.WaitGroup + for i, sig := range sigs[subSet:] { + wg.Add(1) + // test thread safety + go func(i int, sig crypto.Signature) { + defer wg.Done() + index := i + subSet + // test VerifyAndAdd + _, err := aggregator.VerifyAndAdd(ids[index].NodeID, sig, signersData[index].NewestQCView) + // ignore weight as comparing against expected weight is not thread safe + require.NoError(t, err) + }(i, sig) + expectedWeight += ids[i+subSet].Weight + } + + wg.Wait() + actualSignersInfo, aggSig, err := aggregator.Aggregate() + require.NoError(t, err) + require.ElementsMatch(t, signersData[subSet:], actualSignersInfo) + + ok, err := crypto.VerifyBLSSignatureManyMessages(pks[subSet:], aggSig, msgs[subSet:], hashers[subSet:]) + require.NoError(t, err) + require.True(t, ok) + + // add remaining signatures in one thread in order to test the returned weight + for i, sig := range sigs[:subSet] { + weight, err := aggregator.VerifyAndAdd(ids[i].NodeID, sig, signersData[i].NewestQCView) + require.NoError(t, err) + expectedWeight += ids[i].Weight + require.Equal(t, expectedWeight, weight) + // test TotalWeight + require.Equal(t, expectedWeight, aggregator.TotalWeight()) + } + actualSignersInfo, aggSig, err = aggregator.Aggregate() + require.NoError(t, err) + require.ElementsMatch(t, signersData, actualSignersInfo) + + ok, err = crypto.VerifyBLSSignatureManyMessages(pks, aggSig, msgs, hashers) + require.NoError(t, err) + require.True(t, ok) +} + +// TestTimeoutSignatureAggregator_VerifyAndAdd tests behavior of VerifyAndAdd under invalid input data. +func TestTimeoutSignatureAggregator_VerifyAndAdd(t *testing.T) { + signersNum := 20 + + // Unhappy paths + t.Run("invalid signer ID", func(t *testing.T) { + aggregator, _, _, sigs, signersInfo, _, _ := createAggregationData(t, signersNum) + // generate an ID that is not in the node ID list + invalidId := unittest.IdentifierFixture() + + weight, err := aggregator.VerifyAndAdd(invalidId, sigs[0], signersInfo[0].NewestQCView) + require.Equal(t, uint64(0), weight) + require.Equal(t, uint64(0), aggregator.TotalWeight()) + require.True(t, model.IsInvalidSignerError(err)) + }) + + t.Run("duplicate signature", func(t *testing.T) { + aggregator, ids, _, sigs, signersInfo, _, _ := createAggregationData(t, signersNum) + expectedWeight := uint64(0) + // add signatures + for i, sig := range sigs { + weight, err := aggregator.VerifyAndAdd(ids[i].NodeID, sig, signersInfo[i].NewestQCView) + expectedWeight += ids[i].Weight + require.Equal(t, expectedWeight, weight) + require.NoError(t, err) + } + // add same duplicates and test thread safety + var wg sync.WaitGroup + for i, sig := range sigs { + wg.Add(1) + // test thread safety + go func(i int, sig crypto.Signature) { + defer wg.Done() + weight, err := aggregator.VerifyAndAdd(ids[i].NodeID, sigs[i], signersInfo[i].NewestQCView) // same signature for same index + // weight should not change + require.Equal(t, expectedWeight, weight) + require.True(t, model.IsDuplicatedSignerError(err)) + weight, err = aggregator.VerifyAndAdd(ids[i].NodeID, sigs[(i+1)%signersNum], signersInfo[(i+1)%signersNum].NewestQCView) // different signature for same index + // weight should not change + require.Equal(t, expectedWeight, weight) + require.True(t, model.IsDuplicatedSignerError(err)) + weight, err = aggregator.VerifyAndAdd(ids[(i+1)%signersNum].NodeID, sigs[(i+1)%signersNum], signersInfo[(i+1)%signersNum].NewestQCView) // different signature for same index + // weight should not change + require.Equal(t, expectedWeight, weight) + require.True(t, model.IsDuplicatedSignerError(err)) + }(i, sig) + } + wg.Wait() + }) +} + +// TestTimeoutSignatureAggregator_Aggregate tests that Aggregate performs internal checks and +// doesn't produce aggregated signature even when feed with invalid signatures. +func TestTimeoutSignatureAggregator_Aggregate(t *testing.T) { + signersNum := 20 + + t.Run("invalid signature", func(t *testing.T) { + var err error + aggregator, ids, pks, sigs, signersInfo, msgs, hashers := createAggregationData(t, signersNum) + // replace sig with random one + sk := unittest.PrivateKeyFixture(crypto.BLSBLS12381, crypto.KeyGenSeedMinLenECDSAP256) + sigs[0], err = sk.Sign([]byte("dummy"), hashers[0]) + require.NoError(t, err) + + // test VerifyAndAdd + _, err = aggregator.VerifyAndAdd(ids[0].NodeID, sigs[0], signersInfo[0].NewestQCView) + require.ErrorIs(t, err, model.ErrInvalidSignature) + + // add signatures for aggregation including corrupt sigs[0] + expectedWeight := uint64(0) + for i, sig := range sigs { + weight, err := aggregator.VerifyAndAdd(ids[i].NodeID, sig, signersInfo[i].NewestQCView) + if err == nil { + expectedWeight += ids[i].Weight + } + require.Equal(t, expectedWeight, weight) + } + signers, aggSig, err := aggregator.Aggregate() + require.NoError(t, err) + // we should have signers for all signatures except first one since it's invalid + require.Equal(t, len(signers), len(ids)-1) + + ok, err := crypto.VerifyBLSSignatureManyMessages(pks[1:], aggSig, msgs[1:], hashers[1:]) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("aggregating empty set of signatures", func(t *testing.T) { + aggregator, _, _, _, _, _, _ := createAggregationData(t, signersNum) + + // no signatures were added => aggregate should error with + signersData, aggSig, err := aggregator.Aggregate() + require.True(t, model.IsInsufficientSignaturesError(err)) + require.Nil(t, signersData) + require.Nil(t, aggSig) + + // Also, _after_ attempting to add a signature from unknown `signerID`: + // calling `Aggregate()` should error with `model.InsufficientSignaturesError`, + // as still zero signatures are stored. + _, err = aggregator.VerifyAndAdd(unittest.IdentifierFixture(), unittest.SignatureFixture(), 0) + require.True(t, model.IsInvalidSignerError(err)) + + signersData, aggSig, err = aggregator.Aggregate() + require.True(t, model.IsInsufficientSignaturesError(err)) + require.Nil(t, signersData) + require.Nil(t, aggSig) + }) +} diff --git a/consensus/hotstuff/timeoutcollector/factory.go b/consensus/hotstuff/timeoutcollector/factory.go new file mode 100644 index 00000000000..e76c441d1ec --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/factory.go @@ -0,0 +1,97 @@ +package timeoutcollector + +import ( + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" +) + +// TimeoutCollectorFactory implements hotstuff.TimeoutCollectorFactory, it is responsible for creating timeout collector +// for given view. +type TimeoutCollectorFactory struct { + log zerolog.Logger + notifier hotstuff.Consumer + collectorNotifier hotstuff.TimeoutCollectorConsumer + processorFactory hotstuff.TimeoutProcessorFactory +} + +var _ hotstuff.TimeoutCollectorFactory = (*TimeoutCollectorFactory)(nil) + +// NewTimeoutCollectorFactory creates new instance of TimeoutCollectorFactory. +// No error returns are expected during normal operations. +func NewTimeoutCollectorFactory(log zerolog.Logger, + notifier hotstuff.Consumer, + collectorNotifier hotstuff.TimeoutCollectorConsumer, + createProcessor hotstuff.TimeoutProcessorFactory, +) *TimeoutCollectorFactory { + return &TimeoutCollectorFactory{ + log: log, + notifier: notifier, + collectorNotifier: collectorNotifier, + processorFactory: createProcessor, + } +} + +// Create is a factory method to generate a TimeoutCollector for a given view +// Expected error returns during normal operations: +// - model.ErrViewForUnknownEpoch if view is not yet pruned but no epoch containing the given view is known +// +// All other errors should be treated as exceptions. +func (f *TimeoutCollectorFactory) Create(view uint64) (hotstuff.TimeoutCollector, error) { + processor, err := f.processorFactory.Create(view) + if err != nil { + return nil, fmt.Errorf("could not create TimeoutProcessor at view %d: %w", view, err) + } + return NewTimeoutCollector(f.log, view, f.notifier, f.collectorNotifier, processor), nil +} + +// TimeoutProcessorFactory implements hotstuff.TimeoutProcessorFactory, it is responsible for creating timeout processor +// for given view. +type TimeoutProcessorFactory struct { + log zerolog.Logger + committee hotstuff.Replicas + notifier hotstuff.TimeoutCollectorConsumer + validator hotstuff.Validator + domainSeparationTag string +} + +var _ hotstuff.TimeoutProcessorFactory = (*TimeoutProcessorFactory)(nil) + +// NewTimeoutProcessorFactory creates new instance of TimeoutProcessorFactory. +// No error returns are expected during normal operations. +func NewTimeoutProcessorFactory( + log zerolog.Logger, + notifier hotstuff.TimeoutCollectorConsumer, + committee hotstuff.Replicas, + validator hotstuff.Validator, + domainSeparationTag string, +) *TimeoutProcessorFactory { + return &TimeoutProcessorFactory{ + log: log, + committee: committee, + notifier: notifier, + validator: validator, + domainSeparationTag: domainSeparationTag, + } +} + +// Create is a factory method to generate a TimeoutProcessor for a given view +// Expected error returns during normal operations: +// - model.ErrViewForUnknownEpoch no epoch containing the given view is known +// +// All other errors should be treated as exceptions. +func (f *TimeoutProcessorFactory) Create(view uint64) (hotstuff.TimeoutProcessor, error) { + allParticipants, err := f.committee.IdentitiesByEpoch(view) + if err != nil { + return nil, fmt.Errorf("error retrieving consensus participants: %w", err) + } + + sigAggregator, err := NewTimeoutSignatureAggregator(view, allParticipants, f.domainSeparationTag) + if err != nil { + return nil, fmt.Errorf("could not create TimeoutSignatureAggregator at view %d: %w", view, err) + } + + return NewTimeoutProcessor(f.log, f.committee, f.validator, sigAggregator, f.notifier) +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_cache.go b/consensus/hotstuff/timeoutcollector/timeout_cache.go new file mode 100644 index 00000000000..e98df27d022 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_cache.go @@ -0,0 +1,112 @@ +package timeoutcollector + +import ( + "errors" + "sync" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +var ( + // ErrRepeatedTimeout is emitted, when we receive an identical timeout object for the same block + // from the same voter multiple times. This error does _not_ indicate + // equivocation. + ErrRepeatedTimeout = errors.New("duplicated timeout") + ErrTimeoutForIncompatibleView = errors.New("timeout for incompatible view") +) + +// TimeoutObjectsCache maintains a _concurrency safe_ cache of timeouts for one particular +// view. The cache memorizes the order in which the timeouts were received. Timeouts +// are de-duplicated based on the following rules: +// - For each voter (i.e. SignerID), we store the _first_ timeout t0. +// - For any subsequent timeout t, we check whether t.ID() == t0.ID(). +// If this is the case, we consider the timeout a duplicate and drop it. +// If t and t0 have different checksums, the voter is equivocating, and +// we return a model.DoubleTimeoutError. +type TimeoutObjectsCache struct { + lock sync.RWMutex + view uint64 + timeouts map[flow.Identifier]*model.TimeoutObject // signerID -> first timeout +} + +// NewTimeoutObjectsCache instantiates a TimeoutObjectsCache for the given view +func NewTimeoutObjectsCache(view uint64) *TimeoutObjectsCache { + return &TimeoutObjectsCache{ + view: view, + timeouts: make(map[flow.Identifier]*model.TimeoutObject), + } +} + +func (vc *TimeoutObjectsCache) View() uint64 { return vc.view } + +// AddTimeoutObject stores a timeout in the cache. The following errors are expected during +// normal operations: +// - nil: if the timeout was successfully added +// - model.DoubleTimeoutError is returned if the replica is equivocating +// - RepeatedTimeoutErr is returned when adding an _identical_ timeout for the same view from +// the same voter multiple times. +// - TimeoutForIncompatibleViewError is returned if the timeout is for a different view. +// +// When AddTimeoutObject returns an error, the timeout is _not_ stored. +func (vc *TimeoutObjectsCache) AddTimeoutObject(timeout *model.TimeoutObject) error { + if timeout.View != vc.view { + return ErrTimeoutForIncompatibleView + } + vc.lock.Lock() + + // De-duplicated timeouts based on the following rules: + // * For each voter (i.e. SignerID), we store the _first_ t0. + // * For any subsequent timeout t, we check whether t.ID() == t0.ID(). + // If this is the case, we consider the timeout a duplicate and drop it. + // If t and t0 have different checksums, the voter is equivocating, and + // we return a model.DoubleTimeoutError. + firstTimeout, exists := vc.timeouts[timeout.SignerID] + if exists { + vc.lock.Unlock() + // TODO: once we have signer indices, implement Equals methods for QC, TC + // and TimeoutObjects, to avoid the comparatively very expensive ID computation. + if firstTimeout.ID() != timeout.ID() { + return model.NewDoubleTimeoutErrorf(firstTimeout, timeout, "detected timeout equivocation by replica %x at view: %d", timeout.SignerID, vc.view) + } + return ErrRepeatedTimeout + } + vc.timeouts[timeout.SignerID] = timeout + vc.lock.Unlock() + + return nil +} + +// GetTimeoutObject returns the stored timeout for the given `signerID`. Returns: +// - (timeout, true) if a timeout object from signerID is known +// - (nil, false) no timeout object from signerID is known +func (vc *TimeoutObjectsCache) GetTimeoutObject(signerID flow.Identifier) (*model.TimeoutObject, bool) { + vc.lock.RLock() + timeout, exists := vc.timeouts[signerID] // if signerID is unknown, its `Vote` pointer is nil + vc.lock.RUnlock() + return timeout, exists +} + +// Size returns the number of cached timeout objects +func (vc *TimeoutObjectsCache) Size() int { + vc.lock.RLock() + s := len(vc.timeouts) + vc.lock.RUnlock() + return s +} + +// All returns all currently cached timeout objects. Concurrency safe. +func (vc *TimeoutObjectsCache) All() []*model.TimeoutObject { + vc.lock.RLock() + defer vc.lock.RUnlock() + return vc.all() +} + +// all returns all currently cached timeout objects. NOT concurrency safe +func (vc *TimeoutObjectsCache) all() []*model.TimeoutObject { + timeoutObjects := make([]*model.TimeoutObject, 0, len(vc.timeouts)) + for _, t := range vc.timeouts { + timeoutObjects = append(timeoutObjects, t) + } + return timeoutObjects +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_cache_test.go b/consensus/hotstuff/timeoutcollector/timeout_cache_test.go new file mode 100644 index 00000000000..7a627d3945c --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_cache_test.go @@ -0,0 +1,137 @@ +package timeoutcollector + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestTimeoutObjectsCache_View tests that View returns same value that was set by constructor +func TestTimeoutObjectsCache_View(t *testing.T) { + view := uint64(100) + cache := NewTimeoutObjectsCache(view) + require.Equal(t, view, cache.View()) +} + +// TestTimeoutObjectsCache_AddTimeoutObjectRepeatedTimeout tests that AddTimeoutObject skips duplicated timeouts +func TestTimeoutObjectsCache_AddTimeoutObjectRepeatedTimeout(t *testing.T) { + t.Parallel() + + view := uint64(100) + cache := NewTimeoutObjectsCache(view) + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(view)) + + require.NoError(t, cache.AddTimeoutObject(timeout)) + err := cache.AddTimeoutObject(timeout) + require.ErrorIs(t, err, ErrRepeatedTimeout) + require.Len(t, cache.All(), 1) +} + +// TestTimeoutObjectsCache_AddTimeoutObjectIncompatibleView tests that adding timeout with incompatible view results in error +func TestTimeoutObjectsCache_AddTimeoutObjectIncompatibleView(t *testing.T) { + t.Parallel() + + view := uint64(100) + cache := NewTimeoutObjectsCache(view) + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(view + 1)) + err := cache.AddTimeoutObject(timeout) + require.ErrorIs(t, err, ErrTimeoutForIncompatibleView) +} + +// TestTimeoutObjectsCache_GetTimeout tests that GetTimeout method returns the first added timeout +// for a given signer, if any timeout has been added. +func TestTimeoutObjectsCache_GetTimeout(t *testing.T) { + view := uint64(100) + knownTimeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(view)) + doubleTimeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(view), + helper.WithTimeoutObjectSignerID(knownTimeout.SignerID)) + + cache := NewTimeoutObjectsCache(view) + + // unknown timeout + timeout, found := cache.GetTimeoutObject(unittest.IdentifierFixture()) + require.Nil(t, timeout) + require.False(t, found) + + // known timeout + err := cache.AddTimeoutObject(knownTimeout) + require.NoError(t, err) + timeout, found = cache.GetTimeoutObject(knownTimeout.SignerID) + require.Equal(t, knownTimeout, timeout) + require.True(t, found) + + // for a signer ID with a known timeout, the cache should memorize the _first_ encountered timeout + err = cache.AddTimeoutObject(doubleTimeout) + require.True(t, model.IsDoubleTimeoutError(err)) + timeout, found = cache.GetTimeoutObject(doubleTimeout.SignerID) + require.Equal(t, knownTimeout, timeout) + require.True(t, found) +} + +// TestTimeoutObjectsCache_All tests that All returns previously added timeouts. +func TestTimeoutObjectsCache_All(t *testing.T) { + t.Parallel() + + view := uint64(100) + cache := NewTimeoutObjectsCache(view) + expectedTimeouts := make([]*model.TimeoutObject, 5) + for i := range expectedTimeouts { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(view)) + expectedTimeouts[i] = timeout + require.NoError(t, cache.AddTimeoutObject(timeout)) + } + require.ElementsMatch(t, expectedTimeouts, cache.All()) +} + +// BenchmarkAdd measured the time it takes to add `numberTimeouts` concurrently to the TimeoutObjectsCache. +// On MacBook with Intel i7-7820HQ CPU @ 2.90GHz: +// adding 1 million timeouts in total, with 20 threads concurrently, took 0.48s +func BenchmarkAdd(b *testing.B) { + numberTimeouts := 1_000_000 + threads := 20 + + // Setup: create worker routines and timeouts to feed + view := uint64(10) + cache := NewTimeoutObjectsCache(view) + + var start sync.WaitGroup + start.Add(threads) + var done sync.WaitGroup + done.Add(threads) + + n := numberTimeouts / threads + + for ; threads > 0; threads-- { + go func(i int) { + // create timeouts and signal ready + timeouts := make([]model.TimeoutObject, 0, n) + for len(timeouts) < n { + t := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(view)) + timeouts = append(timeouts, *t) + } + start.Done() + + // Wait for last worker routine to signal ready. Then, + // feed all timeouts into cache + start.Wait() + for _, v := range timeouts { + err := cache.AddTimeoutObject(&v) + require.NoError(b, err) + } + done.Done() + }(threads) + } + start.Wait() + t1 := time.Now() + done.Wait() + duration := time.Since(t1) + fmt.Printf("=> adding %d timeouts to Cache took %f seconds\n", cache.Size(), duration.Seconds()) +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector.go b/consensus/hotstuff/timeoutcollector/timeout_collector.go new file mode 100644 index 00000000000..28a9dc6f2d6 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_collector.go @@ -0,0 +1,130 @@ +package timeoutcollector + +import ( + "errors" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/engine/consensus/sealing/counters" +) + +// TimeoutCollector implements logic for collecting timeout objects. Performs deduplication, caching and processing +// of timeouts, delegating those tasks to underlying modules. Emits notifications about verified QCs and TCs, if +// their view is newer than any QC or TC previously known to the TimeoutCollector. +// This module is safe to use in concurrent environment. +type TimeoutCollector struct { + log zerolog.Logger + notifier hotstuff.Consumer + timeoutsCache *TimeoutObjectsCache // cache for tracking double timeout and timeout equivocation + collectorNotifier hotstuff.TimeoutCollectorConsumer + processor hotstuff.TimeoutProcessor + newestReportedQC counters.StrictMonotonousCounter // view of newest QC that was reported + newestReportedTC counters.StrictMonotonousCounter // view of newest TC that was reported +} + +var _ hotstuff.TimeoutCollector = (*TimeoutCollector)(nil) + +// NewTimeoutCollector creates new instance of TimeoutCollector +func NewTimeoutCollector(log zerolog.Logger, + view uint64, + notifier hotstuff.Consumer, + collectorNotifier hotstuff.TimeoutCollectorConsumer, + processor hotstuff.TimeoutProcessor, +) *TimeoutCollector { + return &TimeoutCollector{ + log: log.With(). + Str("component", "hotstuff.timeout_collector"). + Uint64("view", view). + Logger(), + notifier: notifier, + timeoutsCache: NewTimeoutObjectsCache(view), + processor: processor, + collectorNotifier: collectorNotifier, + newestReportedQC: counters.NewMonotonousCounter(0), + newestReportedTC: counters.NewMonotonousCounter(0), + } +} + +// AddTimeout adds a Timeout Object [TO] to the collector. +// When TOs from strictly more than 1/3 of consensus participants (measured by weight) +// were collected, the callback for partial TC will be triggered. +// After collecting TOs from a supermajority, a TC will be created and passed to the EventLoop. +// Expected error returns during normal operations: +// - timeoutcollector.ErrTimeoutForIncompatibleView - submitted timeout for incompatible view +// +// All other exceptions are symptoms of potential state corruption. +func (c *TimeoutCollector) AddTimeout(timeout *model.TimeoutObject) error { + // cache timeout + err := c.timeoutsCache.AddTimeoutObject(timeout) + if err != nil { + if errors.Is(err, ErrRepeatedTimeout) { + return nil + } + if doubleTimeoutErr, isDoubleTimeoutErr := model.AsDoubleTimeoutError(err); isDoubleTimeoutErr { + c.notifier.OnDoubleTimeoutDetected(doubleTimeoutErr.FirstTimeout, doubleTimeoutErr.ConflictingTimeout) + return nil + } + return fmt.Errorf("internal error adding timeout %v to cache for view: %d: %w", timeout.ID(), timeout.View, err) + } + + err = c.processTimeout(timeout) + if err != nil { + return fmt.Errorf("internal error processing TO %v for view: %d: %w", timeout.ID(), timeout.View, err) + } + return nil +} + +// processTimeout delegates TO processing to TimeoutProcessor, handles sentinel errors +// expected errors are handled and reported to notifier. Notifies listeners about validates +// QCs and TCs. +// No errors are expected during normal flow of operations. +func (c *TimeoutCollector) processTimeout(timeout *model.TimeoutObject) error { + err := c.processor.Process(timeout) + if err != nil { + if invalidTimeoutErr, ok := model.AsInvalidTimeoutError(err); ok { + c.notifier.OnInvalidTimeoutDetected(*invalidTimeoutErr) + return nil + } + return fmt.Errorf("internal error while processing timeout: %w", err) + } + + c.notifier.OnTimeoutProcessed(timeout) + + // In the following, we emit notifications about new QCs, if their view is newer than any QC previously + // known to the TimeoutCollector. Note that our implementation only provides weak ordering: + // * Over larger time scales, the emitted events are for statistically increasing views. + // * However, on short time scales there are _no_ monotonicity guarantees w.r.t. the views. + // Explanation: + // While only QCs with strict monotonously increasing views pass the + // `if c.newestReportedQC.Set(timeout.NewestQC.View)` statement, we emit the notification in a separate + // step. Therefore, emitting the notifications is subject to races, where on very short time-scales + // the notifications can be out of order. + // Nevertheless, we note that notifications are only created for QCs that are strictly newer than any other + // known QC at the time we check via the `if ... Set(..)` statement. Thereby, we implement the desired filtering + // behaviour, i.e. that the recipient of the notifications is not spammed by old (or repeated) QCs. + // Reasoning for this approach: + // The current implementation is completely lock-free without noteworthy risk of congestion. For the recipient + // of the notifications, the weak ordering is of no concern, because it anyway is only interested in the newest + // QC. Time-localized disorder is irrelevant, because newer QCs that would arrive later in a strongly ordered + // system can only arrive earlier in our weakly ordered implementation. Hence, if anything, the recipient + // receives the desired information _earlier_ but not later. + if c.newestReportedQC.Set(timeout.NewestQC.View) { + c.collectorNotifier.OnNewQcDiscovered(timeout.NewestQC) + } + // Same explanation for weak ordering of QCs also applies to TCs. + if timeout.LastViewTC != nil { + if c.newestReportedTC.Set(timeout.LastViewTC.View) { + c.collectorNotifier.OnNewTcDiscovered(timeout.LastViewTC) + } + } + + return nil +} + +// View returns view which is associated with this timeout collector +func (c *TimeoutCollector) View() uint64 { + return c.timeoutsCache.View() +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go new file mode 100644 index 00000000000..691209cb179 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go @@ -0,0 +1,194 @@ +package timeoutcollector + +import ( + "errors" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestTimeoutCollector(t *testing.T) { + suite.Run(t, new(TimeoutCollectorTestSuite)) +} + +// TimeoutCollectorTestSuite is a test suite for testing TimeoutCollector. It stores mocked +// state internally for testing behavior. +type TimeoutCollectorTestSuite struct { + suite.Suite + + view uint64 + notifier *mocks.Consumer + collectorNotifier *mocks.TimeoutCollectorConsumer + processor *mocks.TimeoutProcessor + collector *TimeoutCollector +} + +func (s *TimeoutCollectorTestSuite) SetupTest() { + s.view = 1000 + s.notifier = mocks.NewConsumer(s.T()) + s.collectorNotifier = mocks.NewTimeoutCollectorConsumer(s.T()) + s.processor = mocks.NewTimeoutProcessor(s.T()) + + s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Maybe() + s.collectorNotifier.On("OnNewTcDiscovered", mock.Anything).Maybe() + + s.collector = NewTimeoutCollector(unittest.Logger(), s.view, s.notifier, s.collectorNotifier, s.processor) +} + +// TestView tests that `View` returns the same value that was passed in constructor +func (s *TimeoutCollectorTestSuite) TestView() { + require.Equal(s.T(), s.view, s.collector.View()) +} + +// TestAddTimeout_HappyPath tests that process in happy path executed by multiple workers deliver expected results +// all operations should be successful, no errors expected +func (s *TimeoutCollectorTestSuite) TestAddTimeout_HappyPath() { + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view)) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + }() + } + + unittest.AssertReturnsBefore(s.T(), wg.Wait, time.Second) + s.processor.AssertExpectations(s.T()) +} + +// TestAddTimeout_DoubleTimeout tests that submitting two different timeouts for same view ends with reporting +// double timeout to notifier which can be slashed later. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_DoubleTimeout() { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view)) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + + otherTimeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutObjectSignerID(timeout.SignerID)) + + s.notifier.On("OnDoubleTimeoutDetected", timeout, otherTimeout).Once() + + err = s.collector.AddTimeout(otherTimeout) + require.NoError(s.T(), err) + s.notifier.AssertExpectations(s.T()) + s.processor.AssertNumberOfCalls(s.T(), "Process", 1) +} + +// TestAddTimeout_RepeatedTimeout checks that repeated timeouts are silently dropped without any errors. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_RepeatedTimeout() { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view)) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + err = s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + s.processor.AssertNumberOfCalls(s.T(), "Process", 1) +} + +// TestAddTimeout_TimeoutCacheException tests that submitting timeout object for view which is not designated for this +// collector results in ErrTimeoutForIncompatibleView. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_TimeoutCacheException() { + // incompatible view is an exception and not handled by timeout collector + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view + 1)) + err := s.collector.AddTimeout(timeout) + require.ErrorIs(s.T(), err, ErrTimeoutForIncompatibleView) + s.processor.AssertNotCalled(s.T(), "Process") +} + +// TestAddTimeout_InvalidTimeout tests that sentinel errors while processing timeouts are correctly handled and reported +// to notifier, but exceptions are propagated to caller. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_InvalidTimeout() { + s.Run("invalid-timeout", func() { + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view)) + s.processor.On("Process", timeout).Return(model.NewInvalidTimeoutErrorf(timeout, "")).Once() + s.notifier.On("OnInvalidTimeoutDetected", mock.Anything).Run(func(args mock.Arguments) { + invalidTimeoutErr := args.Get(0).(model.InvalidTimeoutError) + require.Equal(s.T(), timeout, invalidTimeoutErr.Timeout) + }).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + + s.notifier.AssertCalled(s.T(), "OnInvalidTimeoutDetected", mock.Anything) + }) + s.Run("process-exception", func() { + exception := errors.New("invalid-signature") + timeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectView(s.view)) + s.processor.On("Process", timeout).Return(exception).Once() + err := s.collector.AddTimeout(timeout) + require.ErrorIs(s.T(), err, exception) + }) +} + +// TestAddTimeout_TONotifications tests that TimeoutCollector in happy path reports the newest discovered QC and TC +func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { + qcCount := 100 + // generate QCs with increasing view numbers + if s.view < uint64(qcCount) { + s.T().Fatal("invalid test configuration") + } + + *s.collectorNotifier = *mocks.NewTimeoutCollectorConsumer(s.T()) + + var highestReportedQC *flow.QuorumCertificate + s.collectorNotifier.On("OnNewQcDiscovered", mock.Anything).Run(func(args mock.Arguments) { + qc := args.Get(0).(*flow.QuorumCertificate) + if highestReportedQC == nil || highestReportedQC.View < qc.View { + highestReportedQC = qc + } + }) + + lastViewTC := helper.MakeTC(helper.WithTCView(s.view - 1)) + s.collectorNotifier.On("OnNewTcDiscovered", lastViewTC).Once() + + timeouts := make([]*model.TimeoutObject, 0, qcCount) + for i := 0; i < qcCount; i++ { + qc := helper.MakeQC(helper.WithQCView(uint64(i))) + timeout := helper.TimeoutObjectFixture(func(timeout *model.TimeoutObject) { + timeout.View = s.view + timeout.NewestQC = qc + timeout.LastViewTC = lastViewTC + }) + timeouts = append(timeouts, timeout) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + } + + expectedHighestQC := timeouts[len(timeouts)-1].NewestQC + + // shuffle timeouts in random order + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(timeouts), func(i, j int) { + timeouts[i], timeouts[j] = timeouts[j], timeouts[i] + }) + + var wg sync.WaitGroup + wg.Add(len(timeouts)) + for _, timeout := range timeouts { + go func(timeout *model.TimeoutObject) { + defer wg.Done() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + }(timeout) + } + wg.Wait() + + require.Equal(s.T(), expectedHighestQC, highestReportedQC) +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_processor.go b/consensus/hotstuff/timeoutcollector/timeout_processor.go new file mode 100644 index 00000000000..60f0e785359 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_processor.go @@ -0,0 +1,315 @@ +package timeoutcollector + +import ( + "errors" + "fmt" + + "github.com/rs/zerolog" + "go.uber.org/atomic" + "golang.org/x/exp/slices" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/tracker" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/module/signature" +) + +// accumulatedWeightTracker tracks one-time event of reaching required weight +// Uses atomic flag to guarantee concurrency safety. +type accumulatedWeightTracker struct { + minRequiredWeight uint64 + done atomic.Bool +} + +func (t *accumulatedWeightTracker) Done() bool { + return t.done.Load() +} + +// Track returns true if `weight` reaches or exceeds `minRequiredWeight` for the _first time_. +// All subsequent calls of `Track` (with any value) return false. +func (t *accumulatedWeightTracker) Track(weight uint64) bool { + if weight < t.minRequiredWeight { + return false + } + return t.done.CompareAndSwap(false, true) +} + +// TimeoutProcessor implements the hotstuff.TimeoutProcessor interface. +// It processes timeout objects broadcast by other replicas of the consensus committee. +// TimeoutProcessor collects TOs for one view, eventually when enough timeout objects are contributed +// TimeoutProcessor will create a timeout certificate which can be used to advance round. +// Concurrency safe. +type TimeoutProcessor struct { + log zerolog.Logger + view uint64 + validator hotstuff.Validator + committee hotstuff.Replicas + sigAggregator hotstuff.TimeoutSignatureAggregator + notifier hotstuff.TimeoutCollectorConsumer + partialTCTracker accumulatedWeightTracker + tcTracker accumulatedWeightTracker + newestQCTracker *tracker.NewestQCTracker +} + +var _ hotstuff.TimeoutProcessor = (*TimeoutProcessor)(nil) + +// NewTimeoutProcessor creates new instance of TimeoutProcessor +// Returns the following expected errors for invalid inputs: +// - model.ErrViewForUnknownEpoch if no epoch containing the given view is known +// +// All other errors should be treated as exceptions. +func NewTimeoutProcessor(log zerolog.Logger, + committee hotstuff.Replicas, + validator hotstuff.Validator, + sigAggregator hotstuff.TimeoutSignatureAggregator, + notifier hotstuff.TimeoutCollectorConsumer, +) (*TimeoutProcessor, error) { + view := sigAggregator.View() + qcThreshold, err := committee.QuorumThresholdForView(view) + if err != nil { + return nil, fmt.Errorf("could not retrieve QC weight threshold for view %d: %w", view, err) + } + timeoutThreshold, err := committee.TimeoutThresholdForView(view) + if err != nil { + return nil, fmt.Errorf("could not retrieve timeout weight threshold for view %d: %w", view, err) + } + return &TimeoutProcessor{ + log: log.With(). + Str("component", "hotstuff.timeout_processor"). + Uint64("view", view). + Logger(), + view: view, + committee: committee, + validator: validator, + notifier: notifier, + partialTCTracker: accumulatedWeightTracker{ + minRequiredWeight: timeoutThreshold, + done: *atomic.NewBool(false), + }, + tcTracker: accumulatedWeightTracker{ + minRequiredWeight: qcThreshold, + done: *atomic.NewBool(false), + }, + sigAggregator: sigAggregator, + newestQCTracker: tracker.NewNewestQCTracker(), + }, nil +} + +// Process performs processing of timeout object in concurrent safe way. This +// function is implemented to be called by multiple goroutines at the same time. +// Design of this function is event driven, as soon as we collect enough weight +// to create a TC or a partial TC we will immediately do so and submit it +// via callback for further processing. +// Expected error returns during normal operations: +// - ErrTimeoutForIncompatibleView - submitted timeout for incompatible view +// - model.InvalidTimeoutError - submitted invalid timeout(invalid structure or invalid signature) +// - model.DuplicatedSignerError if a timeout from the same signer was previously already added +// It does _not necessarily_ imply that the timeout is invalid or the sender is equivocating. +// +// All other errors should be treated as exceptions. +func (p *TimeoutProcessor) Process(timeout *model.TimeoutObject) error { + if p.view != timeout.View { + return fmt.Errorf("received incompatible timeout, expected %d got %d: %w", p.view, timeout.View, ErrTimeoutForIncompatibleView) + } + + if p.tcTracker.Done() { + return nil + } + + err := p.validateTimeout(timeout) + if err != nil { + return fmt.Errorf("validating timeout failed: %w", err) + } + if p.tcTracker.Done() { + return nil + } + + // CAUTION: for correctness it is critical that we update the `newestQCTracker` first, _before_ we add the + // TO's signature to `sigAggregator`. Reasoning: + // * For a valid TC, we require that the TC includes a QC with view ≥ max{TC.NewestQCViews}. + // * The `NewestQCViews` is maintained by `sigAggregator`. + // * Hence, for any view `v ∈ NewestQCViews` that `sigAggregator` knows, a QC with equal or larger view is + // known to `newestQCTracker`. This is guaranteed if and only if `newestQCTracker` is updated first. + p.newestQCTracker.Track(timeout.NewestQC) + + totalWeight, err := p.sigAggregator.VerifyAndAdd(timeout.SignerID, timeout.SigData, timeout.NewestQC.View) + if err != nil { + if model.IsInvalidSignerError(err) { + return model.NewInvalidTimeoutErrorf(timeout, "invalid signer for timeout: %w", err) + } + if errors.Is(err, model.ErrInvalidSignature) { + return model.NewInvalidTimeoutErrorf(timeout, "timeout is from valid signer but has cryptographically invalid signature: %w", err) + } + // model.DuplicatedSignerError is an expected error and just bubbled up the call stack. + // It does _not necessarily_ imply that the timeout is invalid or the sender is equivocating. + return fmt.Errorf("adding signature to aggregator failed: %w", err) + } + p.log.Debug().Msgf("processed timeout, total weight=(%d), required=(%d)", totalWeight, p.tcTracker.minRequiredWeight) + + if p.partialTCTracker.Track(totalWeight) { + p.notifier.OnPartialTcCreated(p.view, p.newestQCTracker.NewestQC(), timeout.LastViewTC) + } + + // Checking of conditions for building TC are satisfied when willBuildTC is true. + // At this point, we have enough signatures to build a TC. Another routine + // might just be at this point. To avoid duplicate work, Track returns true only once. + willBuildTC := p.tcTracker.Track(totalWeight) + if !willBuildTC { + // either we do not have enough timeouts to build a TC, or another thread + // has already passed this gate and created a TC + return nil + } + + tc, err := p.buildTC() + if err != nil { + return fmt.Errorf("internal error constructing TC: %w", err) + } + p.notifier.OnTcConstructedFromTimeouts(tc) + + return nil +} + +// validateTimeout performs validation of timeout object, verifies if timeout is correctly structured +// and included QC and TC is correctly structured and signed. +// ATTENTION: this function does _not_ check whether the TO's `SignerID` is an authorized node nor if +// the signature is valid. These checks happen in signature aggregator. +// Expected error returns during normal operations: +// * model.InvalidTimeoutError - submitted invalid timeout +// All other errors should be treated as exceptions. +func (p *TimeoutProcessor) validateTimeout(timeout *model.TimeoutObject) error { + // 1. check if it's correctly structured + // (a) Every TO must contain a QC + if timeout.NewestQC == nil { + return model.NewInvalidTimeoutErrorf(timeout, "TimeoutObject without QC is invalid") + } + + if timeout.View <= timeout.NewestQC.View { + return model.NewInvalidTimeoutErrorf(timeout, "TO's QC %d cannot be newer than the TO's view %d", + timeout.NewestQC.View, timeout.View) + } + + // (b) If a TC is included, the TC must be for the past round, no matter whether a QC + // for the last round is also included. In some edge cases, a node might observe + // _both_ QC and TC for the previous round, in which case it can include both. + if timeout.LastViewTC != nil { + if timeout.View != timeout.LastViewTC.View+1 { + return model.NewInvalidTimeoutErrorf(timeout, "invalid TC for non-previous view, expected view %d, got view %d", timeout.View-1, timeout.LastViewTC.View) + } + if timeout.NewestQC.View < timeout.LastViewTC.NewestQC.View { + return model.NewInvalidTimeoutErrorf(timeout, "timeout.NewestQC is older (view=%d) than the QC in timeout.LastViewTC (view=%d)", timeout.NewestQC.View, timeout.LastViewTC.NewestQC.View) + } + } + // (c) The TO must contain a proof that sender legitimately entered timeout.View. Transitioning + // to round timeout.View is possible either by observing a QC or a TC for the previous round. + // If no QC is included, we require a TC to be present, which by check (1b) must be for + // the previous round. + lastViewSuccessful := timeout.View == timeout.NewestQC.View+1 + if !lastViewSuccessful { + // The TO's sender did _not_ observe a QC for round timeout.View-1. Hence, it should + // include a TC for the previous round. Otherwise, the TO is invalid. + if timeout.LastViewTC == nil { + return model.NewInvalidTimeoutErrorf(timeout, "timeout must include TC") + } + } + + // 2. Check if QC is valid + err := p.validator.ValidateQC(timeout.NewestQC) + if err != nil { + if model.IsInvalidQCError(err) { + return model.NewInvalidTimeoutErrorf(timeout, "included QC is invalid: %w", err) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the + // Epoch for any QC.View and TC.View we encounter. Receiving a `model.ErrViewForUnknownEpoch` is conceptually impossible, + // i.e. a symptom of an internal bug or invalid bootstrapping information. + return fmt.Errorf("no Epoch information availalbe for QC that was included in TO; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } + return fmt.Errorf("unexpected error when validating QC: %w", err) + } + + // 3. If TC is included, it must be valid + if timeout.LastViewTC != nil { + err = p.validator.ValidateTC(timeout.LastViewTC) + if err != nil { + if model.IsInvalidTCError(err) { + return model.NewInvalidTimeoutErrorf(timeout, "included TC is invalid: %w", err) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the + // Epoch for any QC.View and TC.View we encounter. Receiving a `model.ErrViewForUnknownEpoch` is conceptually impossible, + // i.e. a symptom of an internal bug or invalid bootstrapping information. + return fmt.Errorf("no Epoch information availalbe for TC that was included in TO; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } + return fmt.Errorf("unexpected error when validating TC: %w", err) + } + } + return nil + +} + +// buildTC performs aggregation of signatures when we have collected enough +// weight for building TC. This function is run only once by single worker. +// Any error should be treated as exception. +func (p *TimeoutProcessor) buildTC() (*flow.TimeoutCertificate, error) { + signersData, aggregatedSig, err := p.sigAggregator.Aggregate() + if err != nil { + return nil, fmt.Errorf("could not aggregate multi message signature: %w", err) + } + + // IMPORTANT: To properly verify an aggregated signature included in TC we need to provide list of signers with corresponding + // messages(`TimeoutCertificate.NewestQCViews`) for each signer. If the one-to-once correspondence of view and signer is not maintained, + // it won't be possible to verify the aggregated signature. + // Aggregate returns an unordered set of signers together with additional data. + // Due to implementation specifics of signer indices, the decoding step results in canonically ordered signer ids, which means + // we need to canonically order the respective `newestQCView`, so we can properly map signer to `newestQCView` after decoding. + + // sort data in canonical order + slices.SortFunc(signersData, func(lhs, rhs hotstuff.TimeoutSignerInfo) bool { + return order.IdentifierCanonical(lhs.Signer, rhs.Signer) + }) + + // extract signers and data separately + signers := make([]flow.Identifier, 0, len(signersData)) + newestQCViews := make([]uint64, 0, len(signersData)) + for _, data := range signersData { + signers = append(signers, data.Signer) + newestQCViews = append(newestQCViews, data.NewestQCView) + } + + signerIndices, err := p.signerIndicesFromIdentities(signers) + if err != nil { + return nil, fmt.Errorf("could not encode signer indices: %w", err) + } + + // Note that `newestQC` can have a larger view than any of the views included in `newestQCViews`. + // This is because for a TO currently being processes following two operations are executed in separate steps: + // * updating the `newestQCTracker` with the QC from the TO + // * adding the TO's signature to `sigAggregator` + // Therefore, races are possible, where the `newestQCTracker` already knows of a QC with larger view + // than the data stored in `sigAggregator`. + newestQC := p.newestQCTracker.NewestQC() + + return &flow.TimeoutCertificate{ + View: p.view, + NewestQCViews: newestQCViews, + NewestQC: newestQC, + SignerIndices: signerIndices, + SigData: aggregatedSig, + }, nil +} + +// signerIndicesFromIdentities encodes identities into signer indices. +// Any error should be treated as exception. +func (p *TimeoutProcessor) signerIndicesFromIdentities(signerIDs flow.IdentifierList) ([]byte, error) { + allIdentities, err := p.committee.IdentitiesByEpoch(p.view) + if err != nil { + return nil, fmt.Errorf("could not retrieve identities for view %d: %w", p.view, err) + } + signerIndices, err := signature.EncodeSignersToIndices(allIdentities.NodeIDs(), signerIDs) + if err != nil { + return nil, fmt.Errorf("could not encode signer identifiers to indices: %w", err) + } + return signerIndices, nil +} diff --git a/consensus/hotstuff/timeoutcollector/timeout_processor_test.go b/consensus/hotstuff/timeoutcollector/timeout_processor_test.go new file mode 100644 index 00000000000..b37188c5857 --- /dev/null +++ b/consensus/hotstuff/timeoutcollector/timeout_processor_test.go @@ -0,0 +1,599 @@ +package timeoutcollector + +import ( + "errors" + "fmt" + "math/rand" + "sync" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" + "github.com/onflow/flow-go/consensus/hotstuff/verification" + "github.com/onflow/flow-go/consensus/hotstuff/votecollector" + "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/order" + "github.com/onflow/flow-go/module/local" + msig "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestTimeoutProcessor(t *testing.T) { + suite.Run(t, new(TimeoutProcessorTestSuite)) +} + +// TimeoutProcessorTestSuite is a test suite that holds mocked state for isolated testing of TimeoutProcessor. +type TimeoutProcessorTestSuite struct { + suite.Suite + + participants flow.IdentityList + signer *flow.Identity + view uint64 + sigWeight uint64 + totalWeight atomic.Uint64 + committee *mocks.Replicas + validator *mocks.Validator + sigAggregator *mocks.TimeoutSignatureAggregator + notifier *mocks.TimeoutCollectorConsumer + processor *TimeoutProcessor +} + +func (s *TimeoutProcessorTestSuite) SetupTest() { + var err error + s.sigWeight = 1000 + s.committee = mocks.NewReplicas(s.T()) + s.validator = mocks.NewValidator(s.T()) + s.sigAggregator = mocks.NewTimeoutSignatureAggregator(s.T()) + s.notifier = mocks.NewTimeoutCollectorConsumer(s.T()) + s.participants = unittest.IdentityListFixture(11, unittest.WithWeight(s.sigWeight)).Sort(order.Canonical) + s.signer = s.participants[0] + s.view = (uint64)(rand.Uint32() + 100) + s.totalWeight = *atomic.NewUint64(0) + + s.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(s.participants.TotalWeight()), nil).Maybe() + s.committee.On("TimeoutThresholdForView", mock.Anything).Return(committees.WeightThresholdToTimeout(s.participants.TotalWeight()), nil).Maybe() + s.committee.On("IdentityByEpoch", mock.Anything, mock.Anything).Return(s.signer, nil).Maybe() + s.sigAggregator.On("View").Return(s.view).Maybe() + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + s.totalWeight.Add(s.sigWeight) + }).Return(func(signerID flow.Identifier, sig crypto.Signature, newestQCView uint64) uint64 { + return s.totalWeight.Load() + }, func(signerID flow.Identifier, sig crypto.Signature, newestQCView uint64) error { + return nil + }).Maybe() + s.sigAggregator.On("TotalWeight").Return(func() uint64 { + return s.totalWeight.Load() + }).Maybe() + + s.processor, err = NewTimeoutProcessor( + unittest.Logger(), + s.committee, + s.validator, + s.sigAggregator, + s.notifier, + ) + require.NoError(s.T(), err) +} + +// TimeoutLastViewSuccessfulFixture creates a valid timeout if last view has ended with QC. +func (s *TimeoutProcessorTestSuite) TimeoutLastViewSuccessfulFixture(opts ...func(*model.TimeoutObject)) *model.TimeoutObject { + timeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutNewestQC(helper.MakeQC(helper.WithQCView(s.view-1))), + helper.WithTimeoutLastViewTC(nil), + ) + + for _, opt := range opts { + opt(timeout) + } + + return timeout +} + +// TimeoutLastViewFailedFixture creates a valid timeout if last view has ended with TC. +func (s *TimeoutProcessorTestSuite) TimeoutLastViewFailedFixture(opts ...func(*model.TimeoutObject)) *model.TimeoutObject { + newestQC := helper.MakeQC(helper.WithQCView(s.view - 10)) + timeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutNewestQC(newestQC), + helper.WithTimeoutLastViewTC(helper.MakeTC( + helper.WithTCView(s.view-1), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(newestQC.View))))), + ) + + for _, opt := range opts { + opt(timeout) + } + + return timeout +} + +// TestProcess_TimeoutNotForView tests that TimeoutProcessor accepts only timeouts for the view it was initialized with +// We expect dedicated sentinel errors for timeouts for different views (`ErrTimeoutForIncompatibleView`). +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutNotForView() { + err := s.processor.Process(s.TimeoutLastViewSuccessfulFixture(func(t *model.TimeoutObject) { + t.View++ + })) + require.ErrorIs(s.T(), err, ErrTimeoutForIncompatibleView) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + + s.sigAggregator.AssertNotCalled(s.T(), "Verify") +} + +// TestProcess_TimeoutWithoutQC tests that TimeoutProcessor fails with model.InvalidTimeoutError if +// timeout doesn't contain QC. +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutWithoutQC() { + err := s.processor.Process(s.TimeoutLastViewSuccessfulFixture(func(t *model.TimeoutObject) { + t.NewestQC = nil + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) +} + +// TestProcess_TimeoutNewerHighestQC tests that TimeoutProcessor fails with model.InvalidTimeoutError if +// timeout contains a QC with QC.View > timeout.View, QC can be only with lower view than timeout. +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutNewerHighestQC() { + s.Run("t.View == t.NewestQC.View", func() { + err := s.processor.Process(s.TimeoutLastViewSuccessfulFixture(func(t *model.TimeoutObject) { + t.NewestQC.View = t.View + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + }) + s.Run("t.View < t.NewestQC.View", func() { + err := s.processor.Process(s.TimeoutLastViewSuccessfulFixture(func(t *model.TimeoutObject) { + t.NewestQC.View = t.View + 1 + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + }) +} + +// TestProcess_LastViewTCWrongView tests that TimeoutProcessor fails with model.InvalidTimeoutError if +// timeout contains a proof that sender legitimately entered timeout.View but it has wrong view meaning he used TC from previous rounds. +func (s *TimeoutProcessorTestSuite) TestProcess_LastViewTCWrongView() { + // if TC is included it must have timeout.View == timeout.LastViewTC.View+1 + err := s.processor.Process(s.TimeoutLastViewFailedFixture(func(t *model.TimeoutObject) { + t.LastViewTC.View = t.View - 10 + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) +} + +// TestProcess_LastViewHighestQCInvalidView tests that TimeoutProcessor fails with model.InvalidTimeoutError if +// timeout contains a proof that sender legitimately entered timeout.View but included HighestQC has older view +// than QC included in TC. For honest nodes this shouldn't happen. +func (s *TimeoutProcessorTestSuite) TestProcess_LastViewHighestQCInvalidView() { + err := s.processor.Process(s.TimeoutLastViewFailedFixture(func(t *model.TimeoutObject) { + t.LastViewTC.NewestQC.View = t.NewestQC.View + 1 // TC contains newer QC than Timeout Object + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) +} + +// TestProcess_LastViewTCRequiredButNotPresent tests that TimeoutProcessor fails with model.InvalidTimeoutError if +// timeout must contain a proof that sender legitimately entered timeout.View but doesn't have it. +func (s *TimeoutProcessorTestSuite) TestProcess_LastViewTCRequiredButNotPresent() { + // if last view is not successful(timeout.View != timeout.HighestQC.View+1) then this + // timeout must contain valid timeout.LastViewTC + err := s.processor.Process(s.TimeoutLastViewFailedFixture(func(t *model.TimeoutObject) { + t.LastViewTC = nil + })) + require.True(s.T(), model.IsInvalidTimeoutError(err)) +} + +// TestProcess_IncludedQCInvalid tests that TimeoutProcessor correctly handles validation errors if +// timeout is well-formed but included QC is invalid +func (s *TimeoutProcessorTestSuite) TestProcess_IncludedQCInvalid() { + timeout := s.TimeoutLastViewSuccessfulFixture() + + s.Run("invalid-qc-sentinel", func() { + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(model.InvalidQCError{}).Once() + + err := s.processor.Process(timeout) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + require.True(s.T(), model.IsInvalidQCError(err)) + }) + s.Run("invalid-qc-exception", func() { + exception := errors.New("validate-qc-failed") + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(exception).Once() + + err := s.processor.Process(timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + }) + s.Run("invalid-qc-err-view-for-unknown-epoch", func() { + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(model.ErrViewForUnknownEpoch).Once() + + err := s.processor.Process(timeout) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + require.NotErrorIs(s.T(), err, model.ErrViewForUnknownEpoch) + }) +} + +// TestProcess_IncludedTCInvalid tests that TimeoutProcessor correctly handles validation errors if +// timeout is well-formed but included TC is invalid +func (s *TimeoutProcessorTestSuite) TestProcess_IncludedTCInvalid() { + timeout := s.TimeoutLastViewFailedFixture() + + s.Run("invalid-tc-sentinel", func() { + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil) + s.validator.On("ValidateTC", timeout.LastViewTC).Return(model.InvalidTCError{}) + + err := s.processor.Process(timeout) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + require.True(s.T(), model.IsInvalidTCError(err)) + }) + s.Run("invalid-tc-exception", func() { + exception := errors.New("validate-tc-failed") + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil) + s.validator.On("ValidateTC", timeout.LastViewTC).Return(exception).Once() + + err := s.processor.Process(timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + }) + s.Run("invalid-tc-err-view-for-unknown-epoch", func() { + *s.validator = *mocks.NewValidator(s.T()) + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil) + s.validator.On("ValidateTC", timeout.LastViewTC).Return(model.ErrViewForUnknownEpoch).Once() + + err := s.processor.Process(timeout) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + require.NotErrorIs(s.T(), err, model.ErrViewForUnknownEpoch) + }) +} + +// TestProcess_ValidTimeout tests that processing a valid timeout succeeds without error +func (s *TimeoutProcessorTestSuite) TestProcess_ValidTimeout() { + s.Run("happy-path", func() { + timeout := s.TimeoutLastViewSuccessfulFixture() + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil).Once() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + s.sigAggregator.AssertCalled(s.T(), "VerifyAndAdd", timeout.SignerID, timeout.SigData, timeout.NewestQC.View) + }) + s.Run("recovery-path", func() { + timeout := s.TimeoutLastViewFailedFixture() + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil).Once() + s.validator.On("ValidateTC", timeout.LastViewTC).Return(nil).Once() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + s.sigAggregator.AssertCalled(s.T(), "VerifyAndAdd", timeout.SignerID, timeout.SigData, timeout.NewestQC.View) + }) +} + +// TestProcess_VerifyAndAddFailed tests different scenarios when TimeoutSignatureAggregator fails with error. +// We check all sentinel errors and exceptions in this scenario. +func (s *TimeoutProcessorTestSuite) TestProcess_VerifyAndAddFailed() { + timeout := s.TimeoutLastViewSuccessfulFixture() + s.validator.On("ValidateQC", timeout.NewestQC).Return(nil) + s.Run("invalid-signer", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), model.NewInvalidSignerError(fmt.Errorf(""))).Once() + err := s.processor.Process(timeout) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + require.True(s.T(), model.IsInvalidSignerError(err)) + }) + s.Run("invalid-signature", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), model.ErrInvalidSignature).Once() + err := s.processor.Process(timeout) + require.True(s.T(), model.IsInvalidTimeoutError(err)) + require.ErrorIs(s.T(), err, model.ErrInvalidSignature) + }) + s.Run("duplicated-signer", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), model.NewDuplicatedSignerErrorf("")).Once() + err := s.processor.Process(timeout) + require.True(s.T(), model.IsDuplicatedSignerError(err)) + // this shouldn't be wrapped in invalid timeout + require.False(s.T(), model.IsInvalidTimeoutError(err)) + }) + s.Run("verify-exception", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + exception := errors.New("verify-exception") + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), exception).Once() + err := s.processor.Process(timeout) + require.False(s.T(), model.IsInvalidTimeoutError(err)) + require.ErrorIs(s.T(), err, exception) + }) +} + +// TestProcess_CreatingTC is a test for happy path single threaded signature aggregation and TC creation +// Each replica commits unique timeout object, this object gets processed by TimeoutProcessor. After collecting +// enough weight we expect a TC to be created. All further operations should be no-op, only one TC should be created. +func (s *TimeoutProcessorTestSuite) TestProcess_CreatingTC() { + // consider next situation: + // last successful view was N, after this we weren't able to get a proposal with QC for + // len(participants) views, but in each view QC was created(but not distributed). + // In view N+len(participants) each replica contributes with unique highest QC. + lastSuccessfulQC := helper.MakeQC(helper.WithQCView(s.view - uint64(len(s.participants)))) + lastViewTC := helper.MakeTC(helper.WithTCView(s.view-1), + helper.WithTCNewestQC(lastSuccessfulQC)) + + var highQCViews []uint64 + var timeouts []*model.TimeoutObject + signers := s.participants[1:] + for i, signer := range signers { + qc := helper.MakeQC(helper.WithQCView(lastSuccessfulQC.View + uint64(i+1))) + highQCViews = append(highQCViews, qc.View) + + timeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutNewestQC(qc), + helper.WithTimeoutObjectSignerID(signer.NodeID), + helper.WithTimeoutLastViewTC(lastViewTC), + ) + timeouts = append(timeouts, timeout) + } + + // change tracker to require all except one signer to create TC + s.processor.tcTracker.minRequiredWeight = s.sigWeight * uint64(len(highQCViews)) + + signerIndices, err := msig.EncodeSignersToIndices(s.participants.NodeIDs(), signers.NodeIDs()) + require.NoError(s.T(), err) + expectedSig := crypto.Signature(unittest.RandomBytes(128)) + s.validator.On("ValidateQC", mock.Anything).Return(nil) + s.validator.On("ValidateTC", mock.Anything).Return(nil) + s.notifier.On("OnPartialTcCreated", s.view, mock.Anything, lastViewTC).Return(nil).Once() + s.notifier.On("OnTcConstructedFromTimeouts", mock.Anything).Run(func(args mock.Arguments) { + newestQC := timeouts[len(timeouts)-1].NewestQC + tc := args.Get(0).(*flow.TimeoutCertificate) + // ensure that TC contains correct fields + expectedTC := &flow.TimeoutCertificate{ + View: s.view, + NewestQCViews: highQCViews, + NewestQC: newestQC, + SignerIndices: signerIndices, + SigData: expectedSig, + } + require.Equal(s.T(), expectedTC, tc) + }).Return(nil).Once() + + signersData := make([]hotstuff.TimeoutSignerInfo, 0) + for i, signer := range signers.NodeIDs() { + signersData = append(signersData, hotstuff.TimeoutSignerInfo{ + NewestQCView: highQCViews[i], + Signer: signer, + }) + } + s.sigAggregator.On("Aggregate").Return(signersData, expectedSig, nil) + s.committee.On("IdentitiesByEpoch", s.view).Return(s.participants, nil) + + for _, timeout := range timeouts { + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + } + s.notifier.AssertExpectations(s.T()) + s.sigAggregator.AssertExpectations(s.T()) + + // add extra timeout, make sure we don't create another TC + // should be no-op + timeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutNewestQC(helper.MakeQC(helper.WithQCView(lastSuccessfulQC.View))), + helper.WithTimeoutObjectSignerID(s.participants[0].NodeID), + helper.WithTimeoutLastViewTC(nil), + ) + err = s.processor.Process(timeout) + require.NoError(s.T(), err) + + s.notifier.AssertExpectations(s.T()) + s.validator.AssertExpectations(s.T()) +} + +// TestProcess_ConcurrentCreatingTC tests a scenario where multiple goroutines process timeout at same time, +// we expect only one TC created in this scenario. +func (s *TimeoutProcessorTestSuite) TestProcess_ConcurrentCreatingTC() { + s.validator.On("ValidateQC", mock.Anything).Return(nil) + s.notifier.On("OnPartialTcCreated", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.notifier.On("OnTcConstructedFromTimeouts", mock.Anything).Return(nil).Once() + s.committee.On("IdentitiesByEpoch", mock.Anything).Return(s.participants, nil) + + signersData := make([]hotstuff.TimeoutSignerInfo, 0, len(s.participants)) + for _, signer := range s.participants.NodeIDs() { + signersData = append(signersData, hotstuff.TimeoutSignerInfo{ + NewestQCView: 0, + Signer: signer, + }) + } + // don't care about actual data + s.sigAggregator.On("Aggregate").Return(signersData, crypto.Signature{}, nil) + + var startupWg, shutdownWg sync.WaitGroup + + newestQC := helper.MakeQC(helper.WithQCView(s.view - 1)) + + startupWg.Add(1) + // prepare goroutines, so they are ready to submit a timeout at roughly same time + for i, signer := range s.participants { + shutdownWg.Add(1) + timeout := helper.TimeoutObjectFixture( + helper.WithTimeoutObjectView(s.view), + helper.WithTimeoutNewestQC(newestQC), + helper.WithTimeoutObjectSignerID(signer.NodeID), + helper.WithTimeoutLastViewTC(nil), + ) + go func(i int, timeout *model.TimeoutObject) { + defer shutdownWg.Done() + startupWg.Wait() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + }(i, timeout) + } + + startupWg.Done() + + // wait for all routines to finish + shutdownWg.Wait() +} + +// TestTimeoutProcessor_BuildVerifyTC tests a complete path from creating timeouts to collecting timeouts and then +// building & verifying TC. +// This test emulates the most complex scenario where TC consists of TimeoutObjects that are structurally different. +// Let's consider a case where at some view N consensus committee generated both QC and TC, resulting in nodes differently entering view N+1. +// When constructing TC for view N+1 some replicas will contribute with TO{View:N+1, NewestQC.View: N, LastViewTC: nil} +// while others with TO{View:N+1, NewestQC.View: N-1, LastViewTC: TC{View: N, NewestQC.View: N-1}}. +// This results in multi-message BLS signature with messages picked from set M={N-1,N}. +// We have to be able to construct a valid TC for view N+1 and successfully validate it. +// We start by building a valid QC for view N-1, that will be included in every TimeoutObject at view N. +// Right after we create a valid QC for view N. We need to have valid QCs since TimeoutProcessor performs complete validation of TimeoutObject. +// Then we create a valid cryptographically signed timeout for each signer. Created timeouts are feed to TimeoutProcessor +// which eventually creates a TC after seeing processing enough objects. After we verify if TC was correctly constructed +// and if it doesn't violate protocol rules. At this point we have QC for view N-1, both QC and TC for view N. +// After constructing valid objects we will repeat TC creation process and create a TC for view N+1 where replicas contribute +// with structurally different TimeoutObjects to make sure that TC is correctly built and can be successfully validated. +func TestTimeoutProcessor_BuildVerifyTC(t *testing.T) { + // signers hold objects that are created with private key and can sign votes and proposals + signers := make(map[flow.Identifier]*verification.StakingSigner) + // prepare staking signers, each signer has its own private/public key pair + stakingSigners := unittest.IdentityListFixture(11, func(identity *flow.Identity) { + stakingPriv := unittest.StakingPrivKeyFixture() + identity.StakingPubKey = stakingPriv.PublicKey() + + me, err := local.New(identity, stakingPriv) + require.NoError(t, err) + + signers[identity.NodeID] = verification.NewStakingSigner(me) + }) + // identities must be in canonical order + stakingSigners = stakingSigners.Sort(order.Canonical) + + // utility function which generates a valid timeout for every signer + createTimeouts := func(participants flow.IdentityList, view uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) []*model.TimeoutObject { + timeouts := make([]*model.TimeoutObject, 0, len(participants)) + for _, signer := range participants { + timeout, err := signers[signer.NodeID].CreateTimeout(view, newestQC, lastViewTC) + require.NoError(t, err) + timeouts = append(timeouts, timeout) + } + return timeouts + } + + leader := stakingSigners[0] + + view := uint64(rand.Uint32() + 100) + block := helper.MakeBlock(helper.WithBlockView(view-1), + helper.WithBlockProposer(leader.NodeID)) + + committee := mocks.NewDynamicCommittee(t) + committee.On("IdentitiesByEpoch", mock.Anything).Return(stakingSigners, nil) + committee.On("IdentitiesByBlock", mock.Anything).Return(stakingSigners, nil) + committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(stakingSigners.TotalWeight()), nil) + committee.On("TimeoutThresholdForView", mock.Anything).Return(committees.WeightThresholdToTimeout(stakingSigners.TotalWeight()), nil) + + // create first QC for view N-1, this will be our olderQC + olderQC := createRealQC(t, committee, stakingSigners, signers, block) + // now create a second QC for view N, this will be our newest QC + nextBlock := helper.MakeBlock( + helper.WithBlockView(view), + helper.WithBlockProposer(leader.NodeID), + helper.WithBlockQC(olderQC)) + newestQC := createRealQC(t, committee, stakingSigners, signers, nextBlock) + + // At this point we have created two QCs for round N-1 and N. + // Next step is create a TC for view N. + + // create verifier that will do crypto checks of created TC + verifier := verification.NewStakingVerifier() + // create validator which will do compliance and crypto checks of created TC + validator := hotstuffvalidator.New(committee, verifier) + + var lastViewTC *flow.TimeoutCertificate + onTCCreated := func(args mock.Arguments) { + tc := args.Get(0).(*flow.TimeoutCertificate) + // check if resulted TC is valid + err := validator.ValidateTC(tc) + require.NoError(t, err) + lastViewTC = tc + } + + aggregator, err := NewTimeoutSignatureAggregator(view, stakingSigners, msig.CollectorTimeoutTag) + require.NoError(t, err) + + notifier := mocks.NewTimeoutCollectorConsumer(t) + notifier.On("OnPartialTcCreated", view, olderQC, (*flow.TimeoutCertificate)(nil)).Return().Once() + notifier.On("OnTcConstructedFromTimeouts", mock.Anything).Run(onTCCreated).Return().Once() + processor, err := NewTimeoutProcessor(unittest.Logger(), committee, validator, aggregator, notifier) + require.NoError(t, err) + + // last view was successful, no lastViewTC in this case + timeouts := createTimeouts(stakingSigners, view, olderQC, nil) + for _, timeout := range timeouts { + err := processor.Process(timeout) + require.NoError(t, err) + } + + notifier.AssertExpectations(t) + + // at this point we have created QCs for view N-1 and N additionally a TC for view N, we can create TC for view N+1 + // with timeout objects containing both QC and TC for view N + + aggregator, err = NewTimeoutSignatureAggregator(view+1, stakingSigners, msig.CollectorTimeoutTag) + require.NoError(t, err) + + notifier = mocks.NewTimeoutCollectorConsumer(t) + notifier.On("OnPartialTcCreated", view+1, newestQC, (*flow.TimeoutCertificate)(nil)).Return().Once() + notifier.On("OnTcConstructedFromTimeouts", mock.Anything).Run(onTCCreated).Return().Once() + processor, err = NewTimeoutProcessor(unittest.Logger(), committee, validator, aggregator, notifier) + require.NoError(t, err) + + // part of committee will use QC, another part TC, this will result in aggregated signature consisting + // of two types of messages with views N-1 and N representing the newest QC known to replicas. + timeoutsWithQC := createTimeouts(stakingSigners[:len(stakingSigners)/2], view+1, newestQC, nil) + timeoutsWithTC := createTimeouts(stakingSigners[len(stakingSigners)/2:], view+1, olderQC, lastViewTC) + timeouts = append(timeoutsWithQC, timeoutsWithTC...) + for _, timeout := range timeouts { + err := processor.Process(timeout) + require.NoError(t, err) + } + + notifier.AssertExpectations(t) +} + +// createRealQC is a helper function which generates a properly signed QC with real signatures for given block. +func createRealQC( + t *testing.T, + committee hotstuff.DynamicCommittee, + signers flow.IdentityList, + signerObjects map[flow.Identifier]*verification.StakingSigner, + block *model.Block, +) *flow.QuorumCertificate { + leader := signers[0] + proposal, err := signerObjects[leader.NodeID].CreateProposal(block) + require.NoError(t, err) + + var createdQC *flow.QuorumCertificate + onQCCreated := func(qc *flow.QuorumCertificate) { + createdQC = qc + } + + voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, onQCCreated) + voteProcessor, err := voteProcessorFactory.Create(unittest.Logger(), proposal) + require.NoError(t, err) + + for _, signer := range signers[1:] { + vote, err := signerObjects[signer.NodeID].CreateVote(block) + require.NoError(t, err) + err = voteProcessor.Process(vote) + require.NoError(t, err) + } + + require.NotNil(t, createdQC, "vote processor must create a valid QC at this point") + return createdQC +} diff --git a/consensus/hotstuff/tracker/tracker.go b/consensus/hotstuff/tracker/tracker.go new file mode 100644 index 00000000000..d8a1b9757ae --- /dev/null +++ b/consensus/hotstuff/tracker/tracker.go @@ -0,0 +1,166 @@ +package tracker + +import ( + "unsafe" + + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// NewestQCTracker is a helper structure which keeps track of the newest QC(by view) +// in concurrency safe way. +type NewestQCTracker struct { + newestQC *atomic.UnsafePointer +} + +func NewNewestQCTracker() *NewestQCTracker { + tracker := &NewestQCTracker{ + newestQC: atomic.NewUnsafePointer(unsafe.Pointer(nil)), + } + return tracker +} + +// Track updates local state of NewestQC if the provided instance is newer(by view) +// Concurrently safe +func (t *NewestQCTracker) Track(qc *flow.QuorumCertificate) bool { + // to record the newest value that we have ever seen we need to use loop + // with CAS atomic operation to make sure that we always write the latest value + // in case of shared access to updated value. + for { + // take a snapshot + newestQC := t.NewestQC() + // verify that our update makes sense + if newestQC != nil && newestQC.View >= qc.View { + return false + } + // attempt to install new value, repeat in case of shared update. + if t.newestQC.CompareAndSwap(unsafe.Pointer(newestQC), unsafe.Pointer(qc)) { + return true + } + } +} + +// NewestQC returns the newest QC(by view) tracked. +// Concurrently safe. +func (t *NewestQCTracker) NewestQC() *flow.QuorumCertificate { + return (*flow.QuorumCertificate)(t.newestQC.Load()) +} + +// NewestTCTracker is a helper structure which keeps track of the newest TC(by view) +// in concurrency safe way. +type NewestTCTracker struct { + newestTC *atomic.UnsafePointer +} + +func NewNewestTCTracker() *NewestTCTracker { + tracker := &NewestTCTracker{ + newestTC: atomic.NewUnsafePointer(unsafe.Pointer(nil)), + } + return tracker +} + +// Track updates local state of NewestTC if the provided instance is newer(by view) +// Concurrently safe. +func (t *NewestTCTracker) Track(tc *flow.TimeoutCertificate) bool { + // to record the newest value that we have ever seen we need to use loop + // with CAS atomic operation to make sure that we always write the latest value + // in case of shared access to updated value. + for { + // take a snapshot + newestTC := t.NewestTC() + // verify that our update makes sense + if newestTC != nil && newestTC.View >= tc.View { + return false + } + // attempt to install new value, repeat in case of shared update. + if t.newestTC.CompareAndSwap(unsafe.Pointer(newestTC), unsafe.Pointer(tc)) { + return true + } + } +} + +// NewestTC returns the newest TC(by view) tracked. +// Concurrently safe. +func (t *NewestTCTracker) NewestTC() *flow.TimeoutCertificate { + return (*flow.TimeoutCertificate)(t.newestTC.Load()) +} + +// NewestBlockTracker is a helper structure which keeps track of the newest block (by view) +// in concurrency safe way. +type NewestBlockTracker struct { + newestBlock *atomic.UnsafePointer +} + +func NewNewestBlockTracker() *NewestBlockTracker { + tracker := &NewestBlockTracker{ + newestBlock: atomic.NewUnsafePointer(unsafe.Pointer(nil)), + } + return tracker +} + +// Track updates local state of newestBlock if the provided instance is newer (by view) +// Concurrently safe. +func (t *NewestBlockTracker) Track(block *model.Block) bool { + // to record the newest value that we have ever seen we need to use loop + // with CAS atomic operation to make sure that we always write the latest value + // in case of shared access to updated value. + for { + // take a snapshot + newestBlock := t.NewestBlock() + // verify that our update makes sense + if newestBlock != nil && newestBlock.View >= block.View { + return false + } + // attempt to install new value, repeat in case of shared update. + if t.newestBlock.CompareAndSwap(unsafe.Pointer(newestBlock), unsafe.Pointer(block)) { + return true + } + } +} + +// NewestBlock returns the newest block (by view) tracked. +// Concurrently safe. +func (t *NewestBlockTracker) NewestBlock() *model.Block { + return (*model.Block)(t.newestBlock.Load()) +} + +// NewestPartialTcTracker tracks the newest partial TC (by view) in a concurrency safe way. +type NewestPartialTcTracker struct { + newestPartialTc *atomic.UnsafePointer +} + +func NewNewestPartialTcTracker() *NewestPartialTcTracker { + tracker := &NewestPartialTcTracker{ + newestPartialTc: atomic.NewUnsafePointer(unsafe.Pointer(nil)), + } + return tracker +} + +// Track updates local state of newestPartialTc if the provided instance is newer (by view) +// Concurrently safe. +func (t *NewestPartialTcTracker) Track(partialTc *hotstuff.PartialTcCreated) bool { + // To record the newest value that we have ever seen, we need to use loop + // with CAS atomic operation to make sure that we always write the latest value + // in case of shared access to updated value. + for { + // take a snapshot + newestPartialTc := t.NewestPartialTc() + // verify that our partial TC is from a newer view + if newestPartialTc != nil && newestPartialTc.View >= partialTc.View { + return false + } + // attempt to install new value, repeat in case of shared update. + if t.newestPartialTc.CompareAndSwap(unsafe.Pointer(newestPartialTc), unsafe.Pointer(partialTc)) { + return true + } + } +} + +// NewestPartialTc returns the newest partial TC (by view) tracked. +// Concurrently safe. +func (t *NewestPartialTcTracker) NewestPartialTc() *hotstuff.PartialTcCreated { + return (*hotstuff.PartialTcCreated)(t.newestPartialTc.Load()) +} diff --git a/consensus/hotstuff/tracker/tracker_test.go b/consensus/hotstuff/tracker/tracker_test.go new file mode 100644 index 00000000000..04e5a735097 --- /dev/null +++ b/consensus/hotstuff/tracker/tracker_test.go @@ -0,0 +1,153 @@ +package tracker + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" +) + +// TestNewNewestQCTracker checks that new instance returns nil tracked value. +func TestNewNewestQCTracker(t *testing.T) { + tracker := NewNewestQCTracker() + require.Nil(t, tracker.NewestQC()) +} + +// TestNewestQCTracker_Track this test is needed to make sure that concurrent updates on NewestQCTracker are performed correctly, +// and it always tracks the newest QC, especially in scenario of shared access. This test is structured in a way that it +// starts multiple goroutines that will try to submit their QCs simultaneously to the tracker. Once all goroutines are started +// we will use a wait group to execute all operations as concurrent as possible, after that we will observe if resulted value +// is indeed expected. This test will run multiple times. +func TestNewestQCTracker_Track(t *testing.T) { + tracker := NewNewestQCTracker() + samples := 20 // number of concurrent updates per test case + times := 20 // number of times we run the test case + + // setup initial value + tracker.Track(helper.MakeQC(helper.WithQCView(0))) + + for i := 0; i < times; i++ { + startView := tracker.NewestQC().View + var readyWg, startWg, doneWg sync.WaitGroup + startWg.Add(1) + readyWg.Add(samples) + doneWg.Add(samples) + for s := 0; s < samples; s++ { + qc := helper.MakeQC(helper.WithQCView(startView + uint64(s+1))) + go func(newestQC *flow.QuorumCertificate) { + defer doneWg.Done() + readyWg.Done() + startWg.Wait() + tracker.Track(newestQC) + }(qc) + } + + // wait for all goroutines to be ready + readyWg.Wait() + // since we have waited for all goroutines to be ready this `Done` will start all goroutines + startWg.Done() + // wait for all of them to finish execution + doneWg.Wait() + + // at this point tracker MUST have the newest QC + require.Equal(t, startView+uint64(samples), tracker.NewestQC().View) + } +} + +// TestNewNewestTCTracker checks that new instance returns nil tracked value. +func TestNewNewestTCTracker(t *testing.T) { + tracker := NewNewestTCTracker() + require.Nil(t, tracker.NewestTC()) +} + +// TestNewestTCTracker_Track this test is needed to make sure that concurrent updates on NewestTCTracker are performed correctly, +// and it always tracks the newest TC, especially in scenario of shared access. This test is structured in a way that it +// starts multiple goroutines that will try to submit their TCs simultaneously to the tracker. Once all goroutines are started +// we will use a wait group to execute all operations as concurrent as possible, after that we will observe if resulted value +// is indeed expected. This test will run multiple times. +func TestNewestTCTracker_Track(t *testing.T) { + tracker := NewNewestTCTracker() + samples := 20 + times := 20 + + // setup initial value + tracker.Track(helper.MakeTC(helper.WithTCView(0))) + + for i := 0; i < times; i++ { + startView := tracker.NewestTC().View + var readyWg, startWg, doneWg sync.WaitGroup + startWg.Add(1) + readyWg.Add(samples) + doneWg.Add(samples) + for s := 0; s < samples; s++ { + tc := helper.MakeTC(helper.WithTCView(startView + uint64(s+1))) + go func(newestTC *flow.TimeoutCertificate) { + defer doneWg.Done() + readyWg.Done() + startWg.Wait() + tracker.Track(newestTC) + }(tc) + } + + // wait for all goroutines to be ready + readyWg.Wait() + // since we have waited for all goroutines to be ready this `Done` will start all goroutines + startWg.Done() + // wait for all of them to finish execution + doneWg.Wait() + + // at this point tracker MUST have the newest TC + require.Equal(t, startView+uint64(samples), tracker.NewestTC().View) + } +} + +// TestNewNewestBlockTracker checks that new instance returns nil tracked value. +func TestNewNewestBlockTracker(t *testing.T) { + tracker := NewNewestBlockTracker() + require.Nil(t, tracker.NewestBlock()) +} + +// TestNewestBlockTracker_Track this test is needed to make sure that concurrent updates on NewestBlockTracker are performed correctly, +// and it always tracks the newest block, especially in scenario of shared access. This test is structured in a way that it +// starts multiple goroutines that will try to submit their blocks simultaneously to the tracker. Once all goroutines are started +// we will use a wait group to execute all operations as concurrent as possible, after that we will observe if resulted value +// is indeed expected. This test will run multiple times. +func TestNewestBlockTracker_Track(t *testing.T) { + tracker := NewNewestBlockTracker() + samples := 20 // number of concurrent updates per test case + times := 20 // number of times we run the test case + + // setup initial value + tracker.Track(helper.MakeBlock(helper.WithBlockView(0))) + + for i := 0; i < times; i++ { + startView := tracker.NewestBlock().View + var readyWg, startWg, doneWg sync.WaitGroup + startWg.Add(1) + readyWg.Add(samples) + doneWg.Add(samples) + for s := 0; s < samples; s++ { + block := helper.MakeBlock(helper.WithBlockView(startView + uint64(s+1))) + go func(newestBlock *model.Block) { + defer doneWg.Done() + readyWg.Done() + startWg.Wait() + tracker.Track(newestBlock) + }(block) + } + + // wait for all goroutines to be ready + readyWg.Wait() + // since we have waited for all goroutines to be ready this `Done` will start all goroutines + startWg.Done() + // wait for all of them to finish execution + doneWg.Wait() + + // at this point tracker MUST have the newest block + require.Equal(t, startView+uint64(samples), tracker.NewestBlock().View) + } +} diff --git a/consensus/hotstuff/validator.go b/consensus/hotstuff/validator.go index 5e5c5bd2d4d..17a14ea0603 100644 --- a/consensus/hotstuff/validator.go +++ b/consensus/hotstuff/validator.go @@ -8,19 +8,28 @@ import ( // Validator provides functions to validate QC, proposals and votes. type Validator interface { - // ValidateQC checks the validity of a QC for a given block. + // ValidateQC checks the validity of a QC. // During normal operations, the following error returns are expected: - // * model.InvalidBlockError if the QC is invalid - ValidateQC(qc *flow.QuorumCertificate, block *model.Block) error + // * model.InvalidQCError if the QC is invalid + // * model.ErrViewForUnknownEpoch if the QC refers unknown epoch + ValidateQC(qc *flow.QuorumCertificate) error + + // ValidateTC checks the validity of a TC. + // During normal operations, the following error returns are expected: + // * model.InvalidTCError if the TC is invalid + // * model.ErrViewForUnknownEpoch if the TC refers unknown epoch + ValidateTC(tc *flow.TimeoutCertificate) error // ValidateProposal checks the validity of a proposal. // During normal operations, the following error returns are expected: // * model.InvalidBlockError if the block is invalid + // * model.ErrViewForUnknownEpoch if the proposal refers unknown epoch ValidateProposal(proposal *model.Proposal) error - // ValidateVote checks the validity of a vote for a given block and - // returns the full entity for the voter. During normal operations, + // ValidateVote checks the validity of a vote. + // Returns the full entity for the voter. During normal operations, // the following errors are expected: // * model.InvalidVoteError for invalid votes - ValidateVote(vote *model.Vote, block *model.Block) (*flow.Identity, error) + // * model.ErrViewForUnknownEpoch if the vote refers unknown epoch + ValidateVote(vote *model.Vote) (*flow.Identity, error) } diff --git a/consensus/hotstuff/validator/metrics_wrapper.go b/consensus/hotstuff/validator/metrics_wrapper.go index 037c3af9688..127ca317094 100644 --- a/consensus/hotstuff/validator/metrics_wrapper.go +++ b/consensus/hotstuff/validator/metrics_wrapper.go @@ -26,9 +26,16 @@ func NewMetricsWrapper(validator hotstuff.Validator, metrics module.HotstuffMetr } } -func (w ValidatorMetricsWrapper) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) error { +func (w ValidatorMetricsWrapper) ValidateQC(qc *flow.QuorumCertificate) error { processStart := time.Now() - err := w.validator.ValidateQC(qc, block) + err := w.validator.ValidateQC(qc) + w.metrics.ValidatorProcessingDuration(time.Since(processStart)) + return err +} + +func (w ValidatorMetricsWrapper) ValidateTC(tc *flow.TimeoutCertificate) error { + processStart := time.Now() + err := w.validator.ValidateTC(tc) w.metrics.ValidatorProcessingDuration(time.Since(processStart)) return err } @@ -40,9 +47,9 @@ func (w ValidatorMetricsWrapper) ValidateProposal(proposal *model.Proposal) erro return err } -func (w ValidatorMetricsWrapper) ValidateVote(vote *model.Vote, block *model.Block) (*flow.Identity, error) { +func (w ValidatorMetricsWrapper) ValidateVote(vote *model.Vote) (*flow.Identity, error) { processStart := time.Now() - identity, err := w.validator.ValidateVote(vote, block) + identity, err := w.validator.ValidateVote(vote) w.metrics.ValidatorProcessingDuration(time.Since(processStart)) return identity, err } diff --git a/consensus/hotstuff/validator/validator.go b/consensus/hotstuff/validator/validator.go index 181242b8f43..f52366ad540 100644 --- a/consensus/hotstuff/validator/validator.go +++ b/consensus/hotstuff/validator/validator.go @@ -12,8 +12,7 @@ import ( // Validator is responsible for validating QC, Block and Vote type Validator struct { - committee hotstuff.Committee - forks hotstuff.ForksReader + committee hotstuff.Replicas verifier hotstuff.Verifier } @@ -21,56 +20,150 @@ var _ hotstuff.Validator = (*Validator)(nil) // New creates a new Validator instance func New( - committee hotstuff.Committee, - forks hotstuff.ForksReader, + committee hotstuff.Replicas, verifier hotstuff.Verifier, ) *Validator { return &Validator{ committee: committee, - forks: forks, verifier: verifier, } } -// ValidateQC checks the validity of a QC for a given block. Inputs: -// - qc - the qc to be validated -// - block - the block that the qc is pointing to -// +// ValidateTC validates the TimeoutCertificate `TC`. // During normal operations, the following error returns are expected: -// - model.InvalidBlockError if the QC is invalid -func (v *Validator) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) error { - if qc.BlockID != block.BlockID { - // Sanity check! Failing indicates a bug in the higher-level logic - return fmt.Errorf("qc.BlockID %s doesn't match block's ID %s", qc.BlockID, block.BlockID) +// - model.InvalidTCError if the TC is invalid +// - model.ErrViewForUnknownEpoch if the TC refers unknown epoch +// +// Any other error should be treated as exception +func (v *Validator) ValidateTC(tc *flow.TimeoutCertificate) error { + newestQC := tc.NewestQC + if newestQC == nil { + return newInvalidTCError(tc, fmt.Errorf("TC must include a QC but found nil")) } - if qc.View != block.View { // check view - return newInvalidBlockError(block, fmt.Errorf("qc's View %d doesn't match referenced block's View %d", qc.View, block.View)) + + // The TC's view cannot be smaller than the view of the QC it contains. + // Note: we specifically allow for the TC to have the same view as the highest QC. + // This is useful as a fallback, because it allows replicas other than the designated + // leader to also collect votes and generate a QC. + if tc.View < newestQC.View { + return newInvalidTCError(tc, fmt.Errorf("TC's QC cannot be newer than the TC's view")) } - // Retrieve full Identities of all legitimate consensus participants and the Identities of the qc's signers - // IdentityList returned by hotstuff.Committee contains only legitimate consensus participants for the specified block (must have positive weight) - allParticipants, err := v.committee.Identities(block.BlockID) + // 1. Check if there is super-majority of votes + allParticipants, err := v.committee.IdentitiesByEpoch(tc.View) + if err != nil { + return fmt.Errorf("could not get consensus participants at view %d: %w", tc.View, err) + } + signers, err := signature.DecodeSignerIndicesToIdentities(allParticipants, tc.SignerIndices) if err != nil { - return fmt.Errorf("could not get consensus participants for block %s: %w", block.BlockID, err) + if signature.IsInvalidSignerIndicesError(err) { + return newInvalidTCError(tc, fmt.Errorf("invalid signer indices: %w", err)) + } + // unexpected error + return fmt.Errorf("unexpected internal error decoding signer indices: %w", err) + } + + // determine whether signers reach minimally required weight threshold for consensus + threshold, err := v.committee.QuorumThresholdForView(tc.View) + if err != nil { + return fmt.Errorf("could not get weight threshold for view %d: %w", tc.View, err) + } + if signers.TotalWeight() < threshold { + return newInvalidTCError(tc, fmt.Errorf("tc signers have insufficient weight of %d (required=%d)", signers.TotalWeight(), threshold)) + } + + // Verify multi-message BLS sig of TC, by far the most expensive check + err = v.verifier.VerifyTC(signers, tc.SigData, tc.View, tc.NewestQCViews) + if err != nil { + // Considerations about other errors that `VerifyTC` could return: + // * model.InsufficientSignaturesError: we previously checked the total weight of all signers + // meets the supermajority threshold, which is a _positive_ number. Hence, there must be at + // least one signer. Hence, receiving this error would be a symptom of a fatal internal bug. + switch { + case model.IsInvalidFormatError(err): + return newInvalidTCError(tc, fmt.Errorf("TC's signature data has an invalid structure: %w", err)) + case errors.Is(err, model.ErrInvalidSignature): + return newInvalidTCError(tc, fmt.Errorf("TC contains invalid signature(s): %w", err)) + default: + return fmt.Errorf("cannot verify tc's aggregated signature (tc.View: %d): %w", tc.View, err) + } + } + + // verifying that tc.NewestQC is the QC with the highest view. + // Note: A byzantine TC could include `nil` for tc.NewestQCViews, in which case `tc.NewestQCViews[0]` + // would panic. Though, per API specification `verifier.VerifyTC(…)` should return a `model.InvalidFormatError` + // if `signers` and `tc.NewestQCViews` have different length. Hence, the following code is safe only if it is executed + // 1. _after_ checking the quorum threshold (thereby we guarantee that `signers` is not empty); and + // 2. _after_ `verifier.VerifyTC(…)`, which enforces that `signers` and `tc.NewestQCViews` have identical length. + // Only then we can be sure that `tc.NewestQCViews` cannot be nil. + newestQCView := tc.NewestQCViews[0] + for _, view := range tc.NewestQCViews { + if newestQCView < view { + newestQCView = view + } + } + if newestQCView > tc.NewestQC.View { + return newInvalidTCError(tc, fmt.Errorf("included QC (view=%d) should be equal or higher to highest contributed view: %d", tc.NewestQC.View, newestQCView)) + } + + // Validate QC + err = v.ValidateQC(newestQC) + if err != nil { + if model.IsInvalidQCError(err) { + return newInvalidTCError(tc, fmt.Errorf("invalid QC included in TC: %w", err)) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We require each replica to be bootstrapped with a QC pointing to a finalized block. Consensus safety rules guarantee that + // a QC at least as new as the root QC must be contained in any TC. This is because the TC must include signatures from a + // supermajority of replicas, including at least one honest replica, which attest to their locally highest known QC. Hence, + // any QC included in a TC must be the root QC or newer. Therefore, we should know the Epoch for any QC we encounter. + // receiving a `model.ErrViewForUnknownEpoch` is conceptually impossible, i.e. a symptom of an internal bug or invalid + // bootstrapping information. + return fmt.Errorf("no Epoch information availalbe for QC that was included in TC; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } + return fmt.Errorf("unexpected internal error while verifying the QC included in the TC: %w", err) + } + + return nil +} + +// ValidateQC validates the Quorum Certificate `qc`. +// During normal operations, the following error returns are expected: +// - model.InvalidQCError if the QC is invalid +// - model.ErrViewForUnknownEpoch if the QC refers unknown epoch +// +// Any other error should be treated as exception +func (v *Validator) ValidateQC(qc *flow.QuorumCertificate) error { + // Retrieve the initial identities of consensus participants for this epoch, + // and those that signed the QC. IdentitiesByEpoch contains all nodes that were + // authorized to sign during this epoch. Ejection and dynamic weight adjustments + // are not taken into account here. By using an epoch-static set of authorized + // signers, we can check QC validity without needing all ancestor blocks. + allParticipants, err := v.committee.IdentitiesByEpoch(qc.View) + if err != nil { + return fmt.Errorf("could not get consensus participants at view %d: %w", qc.View, err) } signers, err := signature.DecodeSignerIndicesToIdentities(allParticipants, qc.SignerIndices) if err != nil { if signature.IsInvalidSignerIndicesError(err) { - return newInvalidBlockError(block, fmt.Errorf("invalid signer indices: %w", err)) + return newInvalidQCError(qc, fmt.Errorf("invalid signer indices: %w", err)) } // unexpected error return fmt.Errorf("unexpected internal error decoding signer indices: %w", err) } // determine whether signers reach minimally required weight threshold for consensus - threshold := hotstuff.ComputeWeightThresholdForBuildingQC(allParticipants.TotalWeight()) // compute required weight threshold + threshold, err := v.committee.QuorumThresholdForView(qc.View) + if err != nil { + return fmt.Errorf("could not get weight threshold for view %d: %w", qc.View, err) + } if signers.TotalWeight() < threshold { - return newInvalidBlockError(block, fmt.Errorf("qc signers have insufficient weight of %d (required=%d)", signers.TotalWeight(), threshold)) + return newInvalidQCError(qc, fmt.Errorf("QC signers have insufficient weight of %d (required=%d)", signers.TotalWeight(), threshold)) } - // verify whether the signature bytes are valid for the QC in the context of the protocol state - err = v.verifier.VerifyQC(signers, qc.SigData, block) + // verify whether the signature bytes are valid for the QC + err = v.verifier.VerifyQC(signers, qc.SigData, qc.View, qc.BlockID) if err != nil { // Considerations about other errors that `VerifyQC` could return: // * model.InvalidSignerError: for the time being, we assume that _every_ HotStuff participant @@ -83,9 +176,15 @@ func (v *Validator) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) e // least one signer. Hence, receiving this error would be a symptom of a fatal internal bug. switch { case model.IsInvalidFormatError(err): - return newInvalidBlockError(block, fmt.Errorf("QC's signature data has an invalid structure: %w", err)) + return newInvalidQCError(qc, fmt.Errorf("QC's signature data has an invalid structure: %w", err)) case errors.Is(err, model.ErrInvalidSignature): - return newInvalidBlockError(block, fmt.Errorf("QC contains invalid signature(s): %w", err)) + return newInvalidQCError(qc, fmt.Errorf("QC contains invalid signature(s): %w", err)) + case errors.Is(err, model.ErrViewForUnknownEpoch): + // We have earlier queried the Identities for the QC's view, which must have returned proper values, + // otherwise, we wouldn't reach this code. Therefore, it should be impossible for `verifier.VerifyQC` + // to return ErrViewForUnknownEpoch. To avoid confusion with expected sentinel errors, we only preserve + // the error messages here, but not the error types. + return fmt.Errorf("internal error, as querying identities for view %d succeeded earlier but now the view supposedly belongs to an unknown epoch: %s", qc.View, err.Error()) default: return fmt.Errorf("cannot verify qc's aggregated signature (qc.BlockID: %x): %w", qc.BlockID, err) } @@ -97,12 +196,17 @@ func (v *Validator) ValidateQC(qc *flow.QuorumCertificate, block *model.Block) e // ValidateProposal validates the block proposal // A block is considered as valid if it's a valid extension of existing forks. // Note it doesn't check if it's conflicting with finalized block +// During normal operations, the following error returns are expected: +// - model.InvalidBlockError if the block is invalid +// - model.ErrViewForUnknownEpoch if the proposal refers unknown epoch +// +// Any other error should be treated as exception func (v *Validator) ValidateProposal(proposal *model.Proposal) error { qc := proposal.Block.QC block := proposal.Block // validate the proposer's vote and get his identity - _, err := v.ValidateVote(proposal.ProposerVote(), block) + _, err := v.ValidateVote(proposal.ProposerVote()) if model.IsInvalidVoteError(err) { return newInvalidBlockError(block, fmt.Errorf("invalid proposer signature: %w", err)) } @@ -119,51 +223,88 @@ func (v *Validator) ValidateProposal(proposal *model.Proposal) error { return newInvalidBlockError(block, fmt.Errorf("proposer %s is not leader (%s) for view %d", block.ProposerID, leader, block.View)) } - // check that we have the parent for the proposal - parent, found := v.forks.GetBlock(qc.BlockID) - if !found { - // Forks is _allowed_ to (but obliged to) prune blocks whose view is below the newest finalized block. - if qc.View >= v.forks.FinalizedView() { - // If the parent block is equal or above the finalized view, then Forks should have it. Otherwise, we are missing a block! - return model.MissingBlockError{View: qc.View, BlockID: qc.BlockID} + // The Block must contain a proof that the primary legitimately entered the respective view. + // Transitioning to proposal.Block.View is possible either by observing a QC or a TC for the + // previous round. If and only if the QC is _not_ for the previous round we require a TC for + // the previous view to be present. + lastViewSuccessful := proposal.Block.View == proposal.Block.QC.View+1 + if !lastViewSuccessful { + // check if proposal is correctly structured + if proposal.LastViewTC == nil { + return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC but none is included in block")) + } + + // check if included TC is for previous view + if proposal.Block.View != proposal.LastViewTC.View+1 { + return newInvalidBlockError(block, fmt.Errorf("QC in block is not for previous view, so expecting a TC for view %d but got TC for view %d", proposal.Block.View-1, proposal.LastViewTC.View)) } - // Forks has already pruned the parent block. I.e., we can't validate that the qc matches - // a known (and valid) parent. Nevertheless, we just store this block, because there might already - // exists children of this block, which we could receive later. However, we know for sure that the - // block's fork cannot keep growing anymore because it conflicts with a finalized block. - // TODO: note other components will expect Validator has validated, and might re-validate it, - return model.ErrUnverifiableBlock + // Check if proposal extends either the newest QC specified in the TC, or a newer QC + // in edge cases a leader may construct a TC and QC concurrently such that TC contains + // an older QC - in these case we still want to build on the newest QC, so this case is allowed. + if proposal.Block.QC.View < proposal.LastViewTC.NewestQC.View { + return newInvalidBlockError(block, fmt.Errorf("TC in block contains a newer QC than the block itself, which is a protocol violation")) + } + } else if proposal.LastViewTC != nil { + // last view ended with QC, including TC is a protocol violation + return newInvalidBlockError(block, fmt.Errorf("last view has ended with QC but proposal includes LastViewTC")) } - // validate QC - keep the most expensive the last to check - return v.ValidateQC(qc, parent) -} + // Check signatures, keep the most expensive the last to check -// ValidateVote validates the vote and returns the identity of the voter who signed -// vote - the vote to be validated -// block - the voting block. Assuming the block has been validated. -func (v *Validator) ValidateVote(vote *model.Vote, block *model.Block) (*flow.Identity, error) { - // block hash must match - if vote.BlockID != block.BlockID { - // Sanity check! Failing indicates a bug in the higher-level logic - return nil, fmt.Errorf("wrong block ID. expected (%s), got (%d)", block.BlockID, vote.BlockID) + // check if included QC is valid + err = v.ValidateQC(qc) + if err != nil { + if model.IsInvalidQCError(err) { + return newInvalidBlockError(block, fmt.Errorf("invalid qc included: %w", err)) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the + // Epoch for any QC.View and TC.View we encounter. Receiving a `model.ErrViewForUnknownEpoch` is conceptually impossible, + // i.e. a symptom of an internal bug or invalid bootstrapping information. + return fmt.Errorf("no Epoch information availalbe for QC that was included in proposal; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } + return fmt.Errorf("unexpected error verifying qc: %w", err) } - // view must match with the block's view - if vote.View != block.View { - return nil, newInvalidVoteError(vote, fmt.Errorf("vote's view %d is inconsistent with referenced block (view %d)", vote.View, block.View)) + + if !lastViewSuccessful { + // check if included TC is valid + err = v.ValidateTC(proposal.LastViewTC) + if err != nil { + if model.IsInvalidTCError(err) { + return newInvalidBlockError(block, fmt.Errorf("proposals TC's is not valid: %w", err)) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We require each replica to be bootstrapped with a QC pointing to a finalized block. Therefore, we should know the + // Epoch for any QC.View and TC.View we encounter. Receiving a `model.ErrViewForUnknownEpoch` is conceptually impossible, + // i.e. a symptom of an internal bug or invalid bootstrapping information. + return fmt.Errorf("no Epoch information availalbe for QC that was included in TC; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } + return fmt.Errorf("unexpected internal error while verifying the TC included in block: %w", err) + } } - voter, err := v.committee.Identity(block.BlockID, vote.SignerID) + return nil +} + +// ValidateVote validates the vote and returns the identity of the voter who signed +// vote - the vote to be validated +// During normal operations, the following error returns are expected: +// - model.InvalidVoteError for invalid votes +// - model.ErrViewForUnknownEpoch if the vote refers unknown epoch +// +// Any other error should be treated as exception +func (v *Validator) ValidateVote(vote *model.Vote) (*flow.Identity, error) { + voter, err := v.committee.IdentityByEpoch(vote.View, vote.SignerID) if model.IsInvalidSignerError(err) { return nil, newInvalidVoteError(vote, err) } if err != nil { - return nil, fmt.Errorf("error retrieving voter Identity at block %x: %w", block.BlockID, err) + return nil, fmt.Errorf("error retrieving voter Identity at view %d: %w", vote.View, err) } // check whether the signature data is valid for the vote in the hotstuff context - err = v.verifier.VerifyVote(voter, vote.SigData, block) + err = v.verifier.VerifyVote(voter, vote.SigData, vote.View, vote.BlockID) if err != nil { // Theoretically, `VerifyVote` could also return a `model.InvalidSignerError`. However, // for the time being, we assume that _every_ HotStuff participant is also a member of @@ -173,6 +314,9 @@ func (v *Validator) ValidateVote(vote *model.Vote, block *model.Block) (*flow.Id if model.IsInvalidFormatError(err) || errors.Is(err, model.ErrInvalidSignature) { return nil, newInvalidVoteError(vote, err) } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + return nil, fmt.Errorf("no Epoch information availalbe for vote; symptom of internal bug or invalid bootstrapping information: %s", err.Error()) + } return nil, fmt.Errorf("cannot verify signature for vote (%x): %w", vote.ID(), err) } @@ -187,10 +331,24 @@ func newInvalidBlockError(block *model.Block, err error) error { } } +func newInvalidQCError(qc *flow.QuorumCertificate, err error) error { + return model.InvalidQCError{ + BlockID: qc.BlockID, + View: qc.View, + Err: err, + } +} + +func newInvalidTCError(tc *flow.TimeoutCertificate, err error) error { + return model.InvalidTCError{ + View: tc.View, + Err: err, + } +} + func newInvalidVoteError(vote *model.Vote, err error) error { return model.InvalidVoteError{ - VoteID: vote.ID(), - View: vote.View, - Err: err, + Vote: vote, + Err: err, } } diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index a98ae56cdef..8dbf03736d1 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" @@ -29,6 +30,7 @@ func TestValidateProposal(t *testing.T) { type ProposalSuite struct { suite.Suite participants flow.IdentityList + indices []byte leader *flow.Identity finalized uint64 parent *model.Block @@ -37,8 +39,7 @@ type ProposalSuite struct { proposal *model.Proposal vote *model.Vote voter *flow.Identity - committee *mocks.Committee - forks *mocks.Forks + committee *mocks.Replicas verifier *mocks.Verifier validator *Validator } @@ -55,14 +56,16 @@ func (ps *ProposalSuite) SetupTest() { helper.WithBlockView(ps.finalized), ) - indices, err := signature.EncodeSignersToIndices(ps.participants.NodeIDs(), ps.participants.NodeIDs()) + var err error + + ps.indices, err = signature.EncodeSignersToIndices(ps.participants.NodeIDs(), ps.participants.NodeIDs()) require.NoError(ps.T(), err) ps.block = helper.MakeBlock( helper.WithBlockView(ps.finalized+1), helper.WithBlockProposer(ps.leader.NodeID), helper.WithParentBlock(ps.parent), - helper.WithParentSigners(indices), + helper.WithParentSigners(ps.indices), ) voterIDs, err := signature.DecodeSignerIndicesToIdentifiers(ps.participants.NodeIDs(), ps.block.QC.SignerIndices) @@ -73,36 +76,30 @@ func (ps *ProposalSuite) SetupTest() { ps.vote = ps.proposal.ProposerVote() ps.voter = ps.leader - // set up the mocked hotstuff Committee state - ps.committee = &mocks.Committee{} + // set up the mocked hotstuff Replicas state + ps.committee = &mocks.Replicas{} ps.committee.On("LeaderForView", ps.block.View).Return(ps.leader.NodeID, nil) - ps.committee.On("Identities", mock.Anything).Return( - func(blockID flow.Identifier) flow.IdentityList { + ps.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(ps.participants.TotalWeight()), nil) + ps.committee.On("IdentitiesByEpoch", mock.Anything).Return( + func(_ uint64) flow.IdentityList { return ps.participants }, nil, ) for _, participant := range ps.participants { - ps.committee.On("Identity", mock.Anything, participant.NodeID).Return(participant, nil) + ps.committee.On("IdentityByEpoch", mock.Anything, participant.NodeID).Return(participant, nil) } - // the finalized view is the one of the parent of the - ps.forks = &mocks.Forks{} - ps.forks.On("FinalizedView").Return(ps.finalized) - ps.forks.On("GetBlock", ps.parent.BlockID).Return(ps.parent, true) - ps.forks.On("GetBlock", ps.block.BlockID).Return(ps.block, true) - // set up the mocked verifier ps.verifier = &mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(nil) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(nil).Maybe() + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil).Maybe() // set up the validator with the mocked dependencies - ps.validator = New(ps.committee, ps.forks, ps.verifier) + ps.validator = New(ps.committee, ps.verifier) } func (ps *ProposalSuite) TestProposalOK() { - err := ps.validator.ValidateProposal(ps.proposal) assert.NoError(ps.T(), err, "a valid proposal should be accepted") } @@ -111,8 +108,8 @@ func (ps *ProposalSuite) TestProposalSignatureError() { // change the verifier to error on signature validation with unspecific error *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(nil) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(errors.New("dummy error")) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(nil) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(errors.New("dummy error")) // check that validation now fails err := ps.validator.ValidateProposal(ps.proposal) @@ -126,8 +123,8 @@ func (ps *ProposalSuite) TestProposalSignatureInvalidFormat() { // change the verifier to fail signature validation with InvalidFormatError error *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(nil) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(model.NewInvalidFormatErrorf("")) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(nil) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(model.NewInvalidFormatErrorf("")) // check that validation now fails err := ps.validator.ValidateProposal(ps.proposal) @@ -141,8 +138,8 @@ func (ps *ProposalSuite) TestProposalSignatureInvalid() { // change the verifier to fail signature validation *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(nil) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(model.ErrInvalidSignature) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(nil) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(model.ErrInvalidSignature) // check that validation now fails err := ps.validator.ValidateProposal(ps.proposal) @@ -154,11 +151,11 @@ func (ps *ProposalSuite) TestProposalSignatureInvalid() { func (ps *ProposalSuite) TestProposalWrongLeader() { - // change the hotstuff.Committee to return a different leader - *ps.committee = mocks.Committee{} + // change the hotstuff.Replicas to return a different leader + *ps.committee = mocks.Replicas{} ps.committee.On("LeaderForView", ps.block.View).Return(ps.participants[1].NodeID, nil) for _, participant := range ps.participants { - ps.committee.On("Identity", mock.Anything, participant.NodeID).Return(participant, nil) + ps.committee.On("IdentityByEpoch", mock.Anything, participant.NodeID).Return(participant, nil) } // check that validation fails now @@ -169,72 +166,27 @@ func (ps *ProposalSuite) TestProposalWrongLeader() { assert.True(ps.T(), model.IsInvalidBlockError(err), "if the proposal has wrong proposer, we should generate a invalid error") } -func (ps *ProposalSuite) TestProposalMismatchingView() { - - // change the QC's view to be different from the parent - ps.proposal.Block.QC.View++ - - // check that validation fails now - err := ps.validator.ValidateProposal(ps.proposal) - assert.Error(ps.T(), err, "a proposal with a mismatching QC view should be rejected") - - // check that the error is an invalid proposal error to allow creating slashing challenge - assert.True(ps.T(), model.IsInvalidBlockError(err), "if the QC has a mismatching view, we should generate a invalid error") -} - -func (ps *ProposalSuite) TestProposalMissingParentHigher() { - - // change forks to not find the parent - ps.block.QC.View = ps.finalized - *ps.forks = mocks.Forks{} - ps.forks.On("FinalizedView").Return(ps.finalized) - ps.forks.On("GetBlock", ps.block.QC.BlockID).Return(nil, false) - - // check that validation fails now - err := ps.validator.ValidateProposal(ps.proposal) - assert.Error(ps.T(), err, "a proposal with a missing parent should be rejected") - - // check that the error is a missing block error because we should have the block but we don't - assert.True(ps.T(), model.IsMissingBlockError(err), "if we don't have the proposal parent for a QC above or equal finalized view, we should generate an missing block error") -} - -func (ps *ProposalSuite) TestProposalMissingParentLower() { - - // change forks to not find the parent - ps.block.QC.View = ps.finalized - 1 - *ps.forks = mocks.Forks{} - ps.forks.On("FinalizedView").Return(ps.finalized) - ps.forks.On("GetBlock", ps.block.QC.BlockID).Return(nil, false) - - // check that validation fails now - err := ps.validator.ValidateProposal(ps.proposal) - assert.Error(ps.T(), err, "a proposal with a missing parent should be rejected") - - // check that the error is an unverifiable block because we can't verify the block - assert.True(ps.T(), errors.Is(err, model.ErrUnverifiableBlock), "if we don't have the proposal parent for a QC below finalized view, we should generate an unverifiable block error") -} - // TestProposalQCInvalid checks that Validator handles the verifier's error returns correctly. // In case of `model.InvalidFormatError` and model.ErrInvalidSignature`, we expect the Validator // to recognize those as an invalid QC, i.e. returns an `model.InvalidBlockError`. // In contrast, unexpected exceptions and `model.InvalidSignerError` should _not_ be // interpreted as a sign of an invalid QC. func (ps *ProposalSuite) TestProposalQCInvalid() { - ps.Run("invalid signature", func() { + ps.Run("invalid-signature", func() { *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return( + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return( fmt.Errorf("invalid qc: %w", model.ErrInvalidSignature)) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.True(ps.T(), model.IsInvalidBlockError(err), "if the block's QC signature is invalid, an ErrorInvalidBlock error should be raised") }) - ps.Run("invalid format", func() { + ps.Run("invalid-format", func() { *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(model.NewInvalidFormatErrorf("invalid qc")) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(model.NewInvalidFormatErrorf("invalid qc")) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) @@ -246,11 +198,11 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { // the random beacon committee. Consequently, `InvalidSignerError` should not occur atm. // TODO: if the random beacon committee is a strict subset of the HotStuff committee, // we expect `model.InvalidSignerError` here during normal operations. - ps.Run("invalid signer", func() { + ps.Run("invalid-signer", func() { *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return( + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return( fmt.Errorf("invalid qc: %w", model.NewInvalidSignerErrorf(""))) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) @@ -258,25 +210,37 @@ func (ps *ProposalSuite) TestProposalQCInvalid() { assert.False(ps.T(), model.IsInvalidBlockError(err)) }) - ps.Run("unknown exception", func() { + ps.Run("unknown-exception", func() { exception := errors.New("exception") *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(exception) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(exception) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) // check that validation fails and the failure case is recognized as an invalid block err := ps.validator.ValidateProposal(ps.proposal) assert.ErrorIs(ps.T(), err, exception) assert.False(ps.T(), model.IsInvalidBlockError(err)) }) + + ps.Run("verify-qc-err-view-for-unknown-epoch", func() { + *ps.verifier = mocks.Verifier{} + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(model.ErrViewForUnknownEpoch) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) + + // check that validation fails and the failure is considered internal exception and NOT an InvalidBlock error + err := ps.validator.ValidateProposal(ps.proposal) + assert.Error(ps.T(), err) + assert.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) + assert.False(ps.T(), model.IsInvalidBlockError(err)) + }) } func (ps *ProposalSuite) TestProposalQCError() { // change verifier to fail on QC validation *ps.verifier = mocks.Verifier{} - ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent).Return(fmt.Errorf("some exception")) - ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block).Return(nil) + ps.verifier.On("VerifyQC", ps.voters, ps.block.QC.SigData, ps.parent.View, ps.parent.BlockID).Return(fmt.Errorf("some exception")) + ps.verifier.On("VerifyVote", ps.voter, ps.vote.SigData, ps.block.View, ps.block.BlockID).Return(nil) // check that validation fails now err := ps.validator.ValidateProposal(ps.proposal) @@ -286,6 +250,217 @@ func (ps *ProposalSuite) TestProposalQCError() { assert.False(ps.T(), model.IsInvalidBlockError(err), "if we can't verify the QC, we should not generate a invalid error") } +// TestProposalWithLastViewTC tests different scenarios where last view has ended with TC +// this requires including a valid LastViewTC. +func (ps *ProposalSuite) TestProposalWithLastViewTC() { + // assume all proposals are created by valid leader + ps.verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + ps.committee.On("LeaderForView", mock.Anything).Return(ps.leader.NodeID, nil) + + ps.Run("happy-path", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(ps.block.QC))), + ) + ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(nil).Once() + err := ps.validator.ValidateProposal(proposal) + require.NoError(ps.T(), err) + }) + ps.Run("no-tc", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + // in this case proposal without LastViewTC is considered invalid + ) + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.Run("tc-for-wrong-view", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+10), // LastViewTC.View must be equal to Block.View-1 + helper.WithTCNewestQC(ps.block.QC))), + ) + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.Run("proposal-not-safe-to-extend", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + // proposal is not safe to extend because included QC.View is higher that Block.QC.View + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCView(ps.block.View+1))))), + ) + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyQC") + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.Run("included-tc-highest-qc-not-highest", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(ps.block.QC), + )), + ) + ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, mock.Anything).Return(nil).Once() + + // this is considered an invalid TC, because highest QC's view is not equal to max{NewestQCViews} + proposal.LastViewTC.NewestQCViews[0] = proposal.LastViewTC.NewestQC.View + 1 + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.Run("included-tc-threshold-not-reached", func() { + // TC is signed by only one signer - insufficient to reach weight threshold + insufficientSignerIndices, err := signature.EncodeSignersToIndices(ps.participants.NodeIDs(), ps.participants.NodeIDs()[:1]) + require.NoError(ps.T(), err) + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(insufficientSignerIndices), // one signer is not enough to reach threshold + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(ps.block.QC), + )), + ) + err = ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.Run("included-tc-highest-qc-invalid", func() { + // QC included in TC has view below QC included in proposal + qc := helper.MakeQC( + helper.WithQCView(ps.block.QC.View-1), + helper.WithQCSigners(ps.indices)) + + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(qc))), + ) + ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(nil).Once() + ps.verifier.On("VerifyQC", ps.voters, qc.SigData, + qc.View, qc.BlockID).Return(model.ErrInvalidSignature).Once() + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + }) + ps.Run("verify-qc-err-view-for-unknown-epoch", func() { + newestQC := helper.MakeQC( + helper.WithQCView(ps.block.QC.View-2), + helper.WithQCSigners(ps.indices)) + + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(newestQC))), + ) + ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(nil).Once() + // Validating QC included in TC returns ErrViewForUnknownEpoch + ps.verifier.On("VerifyQC", ps.voters, newestQC.SigData, + newestQC.View, newestQC.BlockID).Return(model.ErrViewForUnknownEpoch).Once() + err := ps.validator.ValidateProposal(proposal) + require.Error(ps.T(), err) + require.False(ps.T(), model.IsInvalidBlockError(err)) + require.False(ps.T(), model.IsInvalidTCError(err)) + require.NotErrorIs(ps.T(), err, model.ErrViewForUnknownEpoch) + }) + ps.Run("included-tc-invalid-sig", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.block.View+2), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithBlockQC(ps.block.QC)), + ), + helper.WithLastViewTC(helper.MakeTC( + helper.WithTCSigners(ps.indices), + helper.WithTCView(ps.block.View+1), + helper.WithTCNewestQC(ps.block.QC))), + ) + ps.verifier.On("VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews).Return(model.ErrInvalidSignature).Once() + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err) && model.IsInvalidTCError(err)) + ps.verifier.AssertCalled(ps.T(), "VerifyTC", ps.voters, []byte(proposal.LastViewTC.SigData), + proposal.LastViewTC.View, proposal.LastViewTC.NewestQCViews) + }) + ps.Run("last-view-successful-but-includes-tc", func() { + proposal := helper.MakeProposal( + helper.WithBlock(helper.MakeBlock( + helper.WithBlockView(ps.finalized+1), + helper.WithBlockProposer(ps.leader.NodeID), + helper.WithParentSigners(ps.indices), + helper.WithParentBlock(ps.parent)), + ), + helper.WithLastViewTC(helper.MakeTC()), + ) + err := ps.validator.ValidateProposal(proposal) + require.True(ps.T(), model.IsInvalidBlockError(err)) + ps.verifier.AssertNotCalled(ps.T(), "VerifyTC") + }) + ps.verifier.AssertExpectations(ps.T()) +} + func TestValidateVote(t *testing.T) { suite.Run(t, new(VoteSuite)) } @@ -295,9 +470,8 @@ type VoteSuite struct { signer *flow.Identity block *model.Block vote *model.Vote - forks *mocks.Forks verifier *mocks.Verifier - committee *mocks.Committee + committee *mocks.Replicas validator *Validator } @@ -317,66 +491,61 @@ func (vs *VoteSuite) SetupTest() { SigData: []byte{}, } - // set up the mocked forks - vs.forks = &mocks.Forks{} - vs.forks.On("GetBlock", vs.block.BlockID).Return(vs.block, true) - // set up the mocked verifier vs.verifier = &mocks.Verifier{} - vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block).Return(nil) + vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block.View, vs.block.BlockID).Return(nil) // the leader for the block view is the correct one - vs.committee = &mocks.Committee{} - vs.committee.On("Identity", mock.Anything, vs.signer.NodeID).Return(vs.signer, nil) + vs.committee = &mocks.Replicas{} + vs.committee.On("IdentityByEpoch", mock.Anything, vs.signer.NodeID).Return(vs.signer, nil) // set up the validator with the mocked dependencies - vs.validator = New(vs.committee, vs.forks, vs.verifier) + vs.validator = New(vs.committee, vs.verifier) } // TestVoteOK checks the happy case, which is the default for the suite func (vs *VoteSuite) TestVoteOK() { - _, err := vs.validator.ValidateVote(vs.vote, vs.block) + _, err := vs.validator.ValidateVote(vs.vote) assert.NoError(vs.T(), err, "a valid vote should be accepted") } -// TestVoteMismatchingView checks that the Validator handles the case where the -// vote contains a mismatching `View` value. In this case, the vote is invalid. -// Hence, we expect the Validator to return a `model.InvalidVoteError`. -func (vs *VoteSuite) TestVoteMismatchingView() { - vs.vote.View++ - - // check that the vote is no longer validated - _, err := vs.validator.ValidateVote(vs.vote, vs.block) - assert.Error(vs.T(), err, "a vote with a mismatching view should be rejected") - - // TODO: this should raise an error that allows a slashing challenge - assert.True(vs.T(), model.IsInvalidVoteError(err), "a mismatching view should create a invalid vote error") -} - // TestVoteSignatureError checks that the Validator does not misinterpret // unexpected exceptions for invalid votes. func (vs *VoteSuite) TestVoteSignatureError() { *vs.verifier = mocks.Verifier{} - vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block).Return(fmt.Errorf("some exception")) + vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block.View, vs.block.BlockID).Return(fmt.Errorf("some exception")) // check that the vote is no longer validated - _, err := vs.validator.ValidateVote(vs.vote, vs.block) + _, err := vs.validator.ValidateVote(vs.vote) assert.Error(vs.T(), err, "a vote with error on signature validation should be rejected") assert.False(vs.T(), model.IsInvalidVoteError(err), "internal exception should not be interpreted as invalid vote") } +// TestVoteVerifyVote_ErrViewForUnknownEpoch tests if ValidateVote correctly handles VerifyVote's ErrViewForUnknownEpoch sentinel error +// Validator shouldn't return a sentinel error here because this behavior is a symptom of internal bug, this behavior is not expected. +func (vs *VoteSuite) TestVoteVerifyVote_ErrViewForUnknownEpoch() { + *vs.verifier = mocks.Verifier{} + vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block.View, vs.block.BlockID).Return(model.ErrViewForUnknownEpoch) + + // check that the vote is no longer validated + _, err := vs.validator.ValidateVote(vs.vote) + assert.Error(vs.T(), err) + assert.False(vs.T(), model.IsInvalidVoteError(err), "internal exception should not be interpreted as invalid vote") + assert.NotErrorIs(vs.T(), err, model.ErrViewForUnknownEpoch, "we don't expect a sentinel error here") +} + // TestVoteInvalidSignerID checks that the Validator correctly handles a vote // with a SignerID that does not correspond to a valid consensus participant. -// In this case, the `hotstuff.Committee` returns a `model.InvalidSignerError`, +// In this case, the `hotstuff.DynamicCommittee` returns a `model.InvalidSignerError`, // which the Validator should recognize as a symptom for an invalid vote. // Hence, we expect the validator to return a `model.InvalidVoteError`. func (vs *VoteSuite) TestVoteInvalidSignerID() { - *vs.committee = mocks.Committee{} - vs.committee.On("Identity", vs.block.BlockID, vs.vote.SignerID).Return(nil, model.NewInvalidSignerErrorf("")) + *vs.committee = mocks.Replicas{} + vs.committee.On("IdentityByEpoch", vs.block.View, vs.vote.SignerID).Return(nil, model.NewInvalidSignerErrorf("")) // A `model.InvalidSignerError` from the committee should be interpreted as // the Vote being invalid, i.e. we expect an InvalidVoteError to be returned - _, err := vs.validator.ValidateVote(vs.vote, vs.block) + _, err := vs.validator.ValidateVote(vs.vote) assert.Error(vs.T(), err, "a vote with unknown SignerID should be rejected") assert.True(vs.T(), model.IsInvalidVoteError(err), "a vote with unknown SignerID should be rejected") } @@ -388,11 +557,11 @@ func (vs *VoteSuite) TestVoteInvalidSignerID() { // Hence, we expect the validator to return a `model.InvalidVoteError`. func (vs *VoteSuite) TestVoteSignatureInvalid() { *vs.verifier = mocks.Verifier{} - vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block).Return(fmt.Errorf("staking sig is invalid: %w", model.ErrInvalidSignature)) + vs.verifier.On("VerifyVote", vs.signer, vs.vote.SigData, vs.block.View, vs.block.BlockID).Return(fmt.Errorf("staking sig is invalid: %w", model.ErrInvalidSignature)) // A `model.ErrInvalidSignature` from the `hotstuff.Verifier` should be interpreted as // the Vote being invalid, i.e. we expect an InvalidVoteError to be returned - _, err := vs.validator.ValidateVote(vs.vote, vs.block) + _, err := vs.validator.ValidateVote(vs.vote) assert.Error(vs.T(), err, "a vote with an invalid signature should be rejected") assert.True(vs.T(), model.IsInvalidVoteError(err), "a vote with an invalid signature should be rejected") } @@ -407,7 +576,7 @@ type QCSuite struct { signers flow.IdentityList block *model.Block qc *flow.QuorumCertificate - committee *mocks.Committee + committee *mocks.Replicas verifier *mocks.Verifier validator *Validator } @@ -430,39 +599,42 @@ func (qs *QCSuite) SetupTest() { qs.qc = helper.MakeQC(helper.WithQCBlock(qs.block), helper.WithQCSigners(indices)) // return the correct participants and identities from view state - qs.committee = &mocks.Committee{} - qs.committee.On("Identities", mock.Anything).Return( - func(blockID flow.Identifier) flow.IdentityList { + qs.committee = &mocks.Replicas{} + qs.committee.On("IdentitiesByEpoch", mock.Anything).Return( + func(_ uint64) flow.IdentityList { return qs.participants }, nil, ) + qs.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(qs.participants.TotalWeight()), nil) // set up the mocked verifier to verify the QC correctly qs.verifier = &mocks.Verifier{} - qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.block).Return(nil) + qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.qc.View, qs.qc.BlockID).Return(nil) // set up the validator with the mocked dependencies - qs.validator = New(qs.committee, nil, qs.verifier) + qs.validator = New(qs.committee, qs.verifier) } // TestQCOK verifies the default happy case func (qs *QCSuite) TestQCOK() { - err := qs.validator.ValidateQC(qs.qc, qs.block) + + // check the default happy case passes + err := qs.validator.ValidateQC(qs.qc) assert.NoError(qs.T(), err, "a valid QC should be accepted") } // TestQCRetrievingParticipantsError tests that validation errors if: // there is an error retrieving identities of consensus participants func (qs *QCSuite) TestQCRetrievingParticipantsError() { - // change the hotstuff.Committee to fail on retrieving participants - *qs.committee = mocks.Committee{} - qs.committee.On("Identities", mock.Anything, mock.Anything).Return(qs.participants, errors.New("FATAL internal error")) + // change the hotstuff.DynamicCommittee to fail on retrieving participants + *qs.committee = mocks.Replicas{} + qs.committee.On("IdentitiesByEpoch", mock.Anything).Return(qs.participants, errors.New("FATAL internal error")) - // verifier should escalate unspecific internal error to surrounding logic, but NOT as ErrorInvalidBlock - err := qs.validator.ValidateQC(qs.qc, qs.block) + // verifier should escalate unspecific internal error to surrounding logic, but NOT as ErrorInvalidQC + err := qs.validator.ValidateQC(qs.qc) assert.Error(qs.T(), err, "unspecific error when retrieving consensus participants should be escalated to surrounding logic") - assert.False(qs.T(), model.IsInvalidBlockError(err), "unspecific internal errors should not result in ErrorInvalidBlock error") + assert.False(qs.T(), model.IsInvalidQCError(err), "unspecific internal errors should not result in ErrorInvalidQC error") } // TestQCSignersError tests that a qc fails validation if: @@ -476,11 +648,11 @@ func (qs *QCSuite) TestQCInsufficientWeight() { qs.qc = helper.MakeQC(helper.WithQCBlock(qs.block), helper.WithQCSigners(indices)) // the QC should not be validated anymore - err = qs.validator.ValidateQC(qs.qc, qs.block) + err = qs.validator.ValidateQC(qs.qc) assert.Error(qs.T(), err, "a QC should be rejected if it has insufficient voted weight") // we should get a threshold error to bubble up for extra info - assert.True(qs.T(), model.IsInvalidBlockError(err), "if there is insufficient voted weight, an invalid block error should be raised") + assert.True(qs.T(), model.IsInvalidQCError(err), "if there is insufficient voted weight, an invalid block error should be raised") } // TestQCSignatureError tests that validation errors if: @@ -489,12 +661,12 @@ func (qs *QCSuite) TestQCSignatureError() { // set up the verifier to fail QC verification *qs.verifier = mocks.Verifier{} - qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.block).Return(errors.New("dummy error")) + qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.qc.View, qs.qc.BlockID).Return(errors.New("dummy error")) - // verifier should escalate unspecific internal error to surrounding logic, but NOT as ErrorInvalidBlock - err := qs.validator.ValidateQC(qs.qc, qs.block) + // verifier should escalate unspecific internal error to surrounding logic, but NOT as ErrorInvalidQC + err := qs.validator.ValidateQC(qs.qc) assert.Error(qs.T(), err, "unspecific sig verification error should be escalated to surrounding logic") - assert.False(qs.T(), model.IsInvalidBlockError(err), "unspecific internal errors should not result in ErrorInvalidBlock error") + assert.False(qs.T(), model.IsInvalidQCError(err), "unspecific internal errors should not result in ErrorInvalidQC error") } // TestQCSignatureInvalid verifies that the Validator correctly handles the model.ErrInvalidSignature. @@ -503,12 +675,22 @@ func (qs *QCSuite) TestQCSignatureError() { func (qs *QCSuite) TestQCSignatureInvalid() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} - qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.block).Return( - fmt.Errorf("invalid signer sig: %w", model.ErrInvalidSignature)) + qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.qc.View, qs.qc.BlockID).Return(fmt.Errorf("invalid qc: %w", model.ErrInvalidSignature)) - // the QC be considered as invalid - err := qs.validator.ValidateQC(qs.qc, qs.block) - assert.True(qs.T(), model.IsInvalidBlockError(err), "if the signature is invalid an ErrorInvalidBlock error should be raised") + // the QC should no longer pass validation + err := qs.validator.ValidateQC(qs.qc) + assert.True(qs.T(), model.IsInvalidQCError(err), "if the signature is invalid an ErrorInvalidQC error should be raised") +} + +// TestQCVerifyQC_ErrViewForUnknownEpoch tests if ValidateQC correctly handles VerifyQC's ErrViewForUnknownEpoch sentinel error +// Validator shouldn't return a sentinel error here because this behavior is a symptom of internal bug, this behavior is not expected. +func (qs *QCSuite) TestQCVerifyQC_ErrViewForUnknownEpoch() { + *qs.verifier = mocks.Verifier{} + qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.qc.View, qs.qc.BlockID).Return(model.ErrViewForUnknownEpoch) + err := qs.validator.ValidateQC(qs.qc) + assert.Error(qs.T(), err) + assert.False(qs.T(), model.IsInvalidQCError(err), "we don't expect a sentinel error here") + assert.NotErrorIs(qs.T(), err, model.ErrViewForUnknownEpoch, "we don't expect a sentinel error here") } // TestQCSignatureInvalidFormat verifies that the Validator correctly handles the model.InvalidFormatError. @@ -517,12 +699,11 @@ func (qs *QCSuite) TestQCSignatureInvalid() { func (qs *QCSuite) TestQCSignatureInvalidFormat() { // change the verifier to fail the QC signature *qs.verifier = mocks.Verifier{} - qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.block).Return( - fmt.Errorf("%w", model.NewInvalidFormatErrorf("invalid sigType"))) + qs.verifier.On("VerifyQC", qs.signers, qs.qc.SigData, qs.qc.View, qs.qc.BlockID).Return(model.NewInvalidFormatErrorf("invalid sigType")) - // the QC be considered as invalid - err := qs.validator.ValidateQC(qs.qc, qs.block) - assert.True(qs.T(), model.IsInvalidBlockError(err), "if the signature has an invalid format, an ErrorInvalidBlock error should be raised") + // the QC should no longer pass validation + err := qs.validator.ValidateQC(qs.qc) + assert.True(qs.T(), model.IsInvalidQCError(err), "if the signature has an invalid format, an ErrorInvalidQC error should be raised") } // TestQCEmptySigners verifies that the Validator correctly handles the model.InsufficientSignaturesError: @@ -532,11 +713,188 @@ func (qs *QCSuite) TestQCSignatureInvalidFormat() { // this error as an invalid QC / invalid block, i.e. it should _not_ return an `InvalidBlockError`. func (qs *QCSuite) TestQCEmptySigners() { *qs.verifier = mocks.Verifier{} - qs.verifier.On("VerifyQC", mock.Anything, qs.qc.SigData, qs.block).Return( + qs.verifier.On("VerifyQC", mock.Anything, qs.qc.SigData, qs.block.View, qs.block.BlockID).Return( fmt.Errorf("%w", model.NewInsufficientSignaturesErrorf(""))) // the Validator should _not_ interpret this as a invalid QC, but as an internal error - err := qs.validator.ValidateQC(qs.qc, qs.block) + err := qs.validator.ValidateQC(qs.qc) assert.True(qs.T(), model.IsInsufficientSignaturesError(err)) // unexpected error should be wrapped and propagated upwards assert.False(qs.T(), model.IsInvalidBlockError(err), err, "should _not_ interpret this as a invalid QC, but as an internal error") } + +func TestValidateTC(t *testing.T) { + suite.Run(t, new(TCSuite)) +} + +type TCSuite struct { + suite.Suite + participants flow.IdentityList + signers flow.IdentityList + indices []byte + block *model.Block + tc *flow.TimeoutCertificate + committee *mocks.DynamicCommittee + verifier *mocks.Verifier + validator *Validator +} + +func (s *TCSuite) SetupTest() { + + // create a list of 10 nodes with 1-weight each + s.participants = unittest.IdentityListFixture(10, + unittest.WithRole(flow.RoleConsensus), + unittest.WithWeight(1), + ) + + // signers are a qualified majority at 7 + s.signers = s.participants[:7] + + var err error + s.indices, err = signature.EncodeSignersToIndices(s.participants.NodeIDs(), s.signers.NodeIDs()) + require.NoError(s.T(), err) + + rand.Seed(time.Now().UnixNano()) + view := uint64(int(rand.Uint32()) + len(s.participants)) + + highQCViews := make([]uint64, 0, len(s.signers)) + for i := range s.signers { + highQCViews = append(highQCViews, view-uint64(i)-1) + } + + rand.Shuffle(len(highQCViews), func(i, j int) { + highQCViews[i], highQCViews[j] = highQCViews[j], highQCViews[i] + }) + + // create a block that has the signers in its QC + parent := helper.MakeBlock(helper.WithBlockView(view - 1)) + s.block = helper.MakeBlock(helper.WithBlockView(view), + helper.WithParentBlock(parent), + helper.WithParentSigners(s.indices)) + s.tc = helper.MakeTC(helper.WithTCNewestQC(s.block.QC), + helper.WithTCView(view+1), + helper.WithTCSigners(s.indices), + helper.WithTCHighQCViews(highQCViews)) + + // return the correct participants and identities from view state + s.committee = &mocks.DynamicCommittee{} + s.committee.On("IdentitiesByEpoch", mock.Anything, mock.Anything).Return( + func(view uint64) flow.IdentityList { + return s.participants + }, + nil, + ) + s.committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(s.participants.TotalWeight()), nil) + + s.verifier = &mocks.Verifier{} + s.verifier.On("VerifyQC", s.signers, s.block.QC.SigData, parent.View, parent.BlockID).Return(nil) + + // set up the validator with the mocked dependencies + s.validator = New(s.committee, s.verifier) +} + +// TestTCOk tests if happy-path returns correct result +func (s *TCSuite) TestTCOk() { + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(nil).Once() + + // check the default happy case passes + err := s.validator.ValidateTC(s.tc) + assert.NoError(s.T(), err, "a valid TC should be accepted") +} + +// TestTCNewestQCFromFuture tests if correct error is returned when included QC is higher than TC's view +func (s *TCSuite) TestTCNewestQCFromFuture() { + // highest QC from future view + s.tc.NewestQC.View = s.tc.View + 1 + err := s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.True(s.T(), model.IsInvalidTCError(err), "if NewestQC.View > TC.View, an ErrorInvalidTC error should be raised") +} + +// TestTCNewestQCIsNotHighest tests if correct error is returned when included QC is not highest +func (s *TCSuite) TestTCNewestQCIsNotHighest() { + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), + s.tc.View, s.tc.NewestQCViews).Return(nil).Once() + + // highest QC view is not equal to max(TONewestQCViews) + s.tc.NewestQCViews[0] = s.tc.NewestQC.View + 1 + err := s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.True(s.T(), model.IsInvalidTCError(err), "if max(highQCViews) != NewestQC.View, an ErrorInvalidTC error should be raised") +} + +// TestTCInvalidSigners tests if correct error is returned when signers are invalid +func (s *TCSuite) TestTCInvalidSigners() { + s.participants = s.participants[1:] // remove participant[0] from the list of valid consensus participant + err := s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.True(s.T(), model.IsInvalidTCError(err), "if some signers are invalid consensus participants, an ErrorInvalidTC error should be raised") +} + +// TestTCThresholdNotReached tests if correct error is returned when TC's singers don't have enough weight +func (s *TCSuite) TestTCThresholdNotReached() { + // signers only have weight 1 out of 10 total (NOT have a supermajority) + s.signers = s.participants[:1] + indices, err := signature.EncodeSignersToIndices(s.participants.NodeIDs(), s.signers.NodeIDs()) + require.NoError(s.T(), err) + + s.tc.SignerIndices = indices + + // adjust signers to be less than total weight + err = s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.True(s.T(), model.IsInvalidTCError(err), "if signers don't have enough weight, an ErrorInvalidTC error should be raised") +} + +// TestTCInvalidNewestQC tests if correct error is returned when included highest QC is invalid +func (s *TCSuite) TestTCInvalidNewestQC() { + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(nil).Once() + s.verifier.On("VerifyQC", s.signers, s.tc.NewestQC.SigData, s.tc.NewestQC.View, s.tc.NewestQC.BlockID).Return(model.NewInvalidFormatErrorf("invalid qc")).Once() + err := s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.True(s.T(), model.IsInvalidTCError(err), "if included QC is invalid, an ErrorInvalidTC error should be raised") +} + +// TestTCVerifyQC_ErrViewForUnknownEpoch tests if ValidateTC correctly handles VerifyQC's ErrViewForUnknownEpoch sentinel error +// Validator shouldn't return a sentinel error here because this behavior is a symptom of internal bug, this behavior is not expected. +func (s *TCSuite) TestTCVerifyQC_ErrViewForUnknownEpoch() { + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(nil).Once() + s.verifier.On("VerifyQC", s.signers, s.tc.NewestQC.SigData, s.tc.NewestQC.View, s.tc.NewestQC.BlockID).Return(model.ErrViewForUnknownEpoch).Once() + err := s.validator.ValidateTC(s.tc) // the QC should not be validated anymore + assert.Error(s.T(), err) + assert.False(s.T(), model.IsInvalidTCError(err), "we don't expect a sentinel error here") + assert.NotErrorIs(s.T(), err, model.ErrViewForUnknownEpoch, "we don't expect a sentinel error here") +} + +// TestTCInvalidSignature tests a few scenarios when the signature is invalid or TC signers is malformed +func (s *TCSuite) TestTCInvalidSignature() { + s.Run("insufficient-signatures", func() { + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.verifier.On("VerifyTC", mock.Anything, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(model.NewInsufficientSignaturesErrorf("")).Once() + + // the Validator should _not_ interpret this as an invalid TC, but as an internal error + err := s.validator.ValidateTC(s.tc) + assert.True(s.T(), model.IsInsufficientSignaturesError(err)) // unexpected error should be wrapped and propagated upwards + assert.False(s.T(), model.IsInvalidTCError(err), err, "should _not_ interpret this as a invalid TC, but as an internal error") + }) + s.Run("invalid-format", func() { + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(model.NewInvalidFormatErrorf("")).Once() + err := s.validator.ValidateTC(s.tc) + assert.True(s.T(), model.IsInvalidTCError(err), "if included TC's inputs are invalid, an ErrorInvalidTC error should be raised") + }) + s.Run("invalid-signature", func() { + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(model.ErrInvalidSignature).Once() + err := s.validator.ValidateTC(s.tc) + assert.True(s.T(), model.IsInvalidTCError(err), "if included TC's signature is invalid, an ErrorInvalidTC error should be raised") + }) + s.Run("verify-sig-exception", func() { + exception := errors.New("verify-sig-exception") + *s.verifier = mocks.Verifier{} + s.verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.verifier.On("VerifyTC", s.signers, []byte(s.tc.SigData), s.tc.View, s.tc.NewestQCViews).Return(exception).Once() + err := s.validator.ValidateTC(s.tc) + assert.ErrorIs(s.T(), err, exception, "if included TC's signature is invalid, an exception should be propagated") + assert.False(s.T(), model.IsInvalidTCError(err)) + }) +} diff --git a/consensus/hotstuff/verification/combined_signer_v2.go b/consensus/hotstuff/verification/combined_signer_v2.go index 6946b49c1fd..68138808f01 100644 --- a/consensus/hotstuff/verification/combined_signer_v2.go +++ b/consensus/hotstuff/verification/combined_signer_v2.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto/hash" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/signature" + msig "github.com/onflow/flow-go/module/signature" ) // CombinedSigner creates votes for the main consensus. @@ -21,12 +23,15 @@ import ( // The difference between V2 and V3 is that V2 will sign 2 sigs, whereas // V3 only sign 1 sig. type CombinedSigner struct { - staking module.Local - stakingHasher hash.Hasher - beaconKeyStore module.RandomBeaconKeyStore - beaconHasher hash.Hasher + staking module.Local + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher + beaconKeyStore module.RandomBeaconKeyStore + beaconHasher hash.Hasher } +var _ hotstuff.Signer = (*CombinedSigner)(nil) + // NewCombinedSigner creates a new combined signer with the given dependencies: // - the staking signer is used to create and verify aggregatable signatures for Hotstuff // - the beaconKeyStore is used to get threshold-signers by epoch/view; @@ -37,10 +42,11 @@ func NewCombinedSigner( ) *CombinedSigner { sc := &CombinedSigner{ - staking: staking, - stakingHasher: signature.NewBLSHasher(signature.ConsensusVoteTag), - beaconKeyStore: beaconKeyStore, - beaconHasher: signature.NewBLSHasher(signature.RandomBeaconTag), + staking: staking, + stakingHasher: msig.NewBLSHasher(msig.ConsensusVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.ConsensusTimeoutTag), + beaconKeyStore: beaconKeyStore, + beaconHasher: msig.NewBLSHasher(msig.RandomBeaconTag), } return sc } @@ -88,6 +94,26 @@ func (c *CombinedSigner) CreateVote(block *model.Block) (*model.Vote, error) { return vote, nil } +// CreateTimeout will create a signed timeout object for the given view. +// Timeout objects are only signed with the staking key (not beacon key). +func (c *CombinedSigner) CreateTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + // create timeout object specific message + msg := MakeTimeoutMessage(curView, newestQC.View) + sigData, err := c.staking.Sign(msg, c.timeoutObjectHasher) + if err != nil { + return nil, fmt.Errorf("could not generate signature for timeout object at view %d: %w", curView, err) + } + + timeout := &model.TimeoutObject{ + View: curView, + NewestQC: newestQC, + LastViewTC: lastViewTC, + SignerID: c.staking.NodeID(), + SigData: sigData, + } + return timeout, nil +} + // genSigData generates the signature data for our local node for the given block. // It returns: // - (stakingSig, nil) if there is no random beacon private key. @@ -118,5 +144,5 @@ func (c *CombinedSigner) genSigData(block *model.Block) ([]byte, error) { return nil, fmt.Errorf("could not generate beacon signature: %w", err) } - return signature.EncodeDoubleSig(stakingSig, beaconShare), nil + return msig.EncodeDoubleSig(stakingSig, beaconShare), nil } diff --git a/consensus/hotstuff/verification/combined_signer_v2_test.go b/consensus/hotstuff/verification/combined_signer_v2_test.go index f084aec684c..eb34847dd5e 100644 --- a/consensus/hotstuff/verification/combined_signer_v2_test.go +++ b/consensus/hotstuff/verification/combined_signer_v2_test.go @@ -32,7 +32,7 @@ func TestCombinedSignWithDKGKey(t *testing.T) { fblock := unittest.BlockFixture() fblock.Header.ProposerID = identities[0].NodeID fblock.Header.View = view - block := model.BlockFromFlow(fblock.Header, 10) + block := model.BlockFromFlow(fblock.Header) signerID := fblock.Header.ProposerID epochCounter := uint64(3) @@ -57,7 +57,7 @@ func TestCombinedSignWithDKGKey(t *testing.T) { dkg := &mocks.DKG{} dkg.On("KeyShare", signerID).Return(pk, nil) - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} committee.On("DKG", mock.Anything).Return(dkg, nil) packer := signature.NewConsensusSigDataPacker(committee) @@ -68,7 +68,7 @@ func TestCombinedSignWithDKGKey(t *testing.T) { require.NoError(t, err) vote := proposal.ProposerVote() - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) // check that a created proposal's signature is a combined staking sig and random beacon sig @@ -86,31 +86,31 @@ func TestCombinedSignWithDKGKey(t *testing.T) { vote, err = signer.CreateVote(block) require.NoError(t, err) - err = verifier.VerifyVote(nodeID, vote.SigData, block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) // vote on different block should be invalid blockWrongID := *block blockWrongID.BlockID[0]++ - err = verifier.VerifyVote(nodeID, vote.SigData, &blockWrongID) + err = verifier.VerifyVote(nodeID, vote.SigData, blockWrongID.View, blockWrongID.BlockID) require.ErrorIs(t, err, model.ErrInvalidSignature) // vote with a wrong view should be invalid blockWrongView := *block blockWrongView.View++ - err = verifier.VerifyVote(nodeID, vote.SigData, &blockWrongView) + err = verifier.VerifyVote(nodeID, vote.SigData, blockWrongID.View, blockWrongID.BlockID) require.ErrorIs(t, err, model.ErrInvalidSignature) // vote by different signer should be invalid wrongVoter := identities[1] wrongVoter.StakingPubKey = unittest.StakingPrivKeyFixture().PublicKey() - err = verifier.VerifyVote(wrongVoter, vote.SigData, block) + err = verifier.VerifyVote(wrongVoter, vote.SigData, block.View, block.BlockID) require.ErrorIs(t, err, model.ErrInvalidSignature) // vote with changed signature should be invalid brokenSig := append([]byte{}, vote.SigData...) // copy brokenSig[4]++ - err = verifier.VerifyVote(nodeID, brokenSig, block) + err = verifier.VerifyVote(nodeID, brokenSig, block.View, block.BlockID) require.ErrorIs(t, err, model.ErrInvalidSignature) // Vote from a node that is _not_ part of the Random Beacon committee should be rejected. @@ -118,7 +118,7 @@ func TestCombinedSignWithDKGKey(t *testing.T) { // as a sign of an invalid vote and wraps it into a `model.InvalidSignerError`. *dkg = mocks.DKG{} // overwrite DKG mock with a new one dkg.On("KeyShare", signerID).Return(nil, protocol.IdentityNotFoundError{NodeID: signerID}) - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.True(t, model.IsInvalidSignerError(err)) } @@ -132,7 +132,7 @@ func TestCombinedSignWithNoDKGKey(t *testing.T) { fblock := unittest.BlockFixture() fblock.Header.View = view - block := model.BlockFromFlow(fblock.Header, 10) + block := model.BlockFromFlow(fblock.Header) signerID := fblock.Header.ProposerID epochCounter := uint64(3) @@ -157,7 +157,7 @@ func TestCombinedSignWithNoDKGKey(t *testing.T) { dkg := &mocks.DKG{} dkg.On("KeyShare", signerID).Return(pk, nil) - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} // even if the node failed DKG, and has no random beacon private key, // but other nodes, who completed and succeeded DKG, have a public key // for this failed node, which can be used to verify signature from @@ -171,7 +171,7 @@ func TestCombinedSignWithNoDKGKey(t *testing.T) { require.NoError(t, err) vote := proposal.ProposerVote() - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) // As the proposer does not have a Random Beacon Key, it should sign solely with its staking key. @@ -189,17 +189,17 @@ func TestCombinedSignWithNoDKGKey(t *testing.T) { // any sub-components, because some (e.g. `crypto.AggregateBLSPublicKeys`) don't provide sufficient // sentinel errors to distinguish between internal problems and external byzantine inputs. func Test_VerifyQC_EmptySigners(t *testing.T) { - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} packer := signature.NewConsensusSigDataPacker(committee) verifier := NewCombinedVerifier(committee, packer) header := unittest.BlockHeaderFixture() - block := model.BlockFromFlow(header, header.View-1) + block := model.BlockFromFlow(header) sigData := unittest.QCSigDataFixture() - err := verifier.VerifyQC([]*flow.Identity{}, sigData, block) + err := verifier.VerifyQC([]*flow.Identity{}, sigData, block.View, block.BlockID) require.True(t, model.IsInsufficientSignaturesError(err)) - err = verifier.VerifyQC(nil, sigData, block) + err = verifier.VerifyQC(nil, sigData, block.View, block.BlockID) require.True(t, model.IsInsufficientSignaturesError(err)) } diff --git a/consensus/hotstuff/verification/combined_signer_v3.go b/consensus/hotstuff/verification/combined_signer_v3.go index 8f87c7c07b0..99126871165 100644 --- a/consensus/hotstuff/verification/combined_signer_v3.go +++ b/consensus/hotstuff/verification/combined_signer_v3.go @@ -4,11 +4,13 @@ import ( "errors" "fmt" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/model/encoding" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/signature" + msig "github.com/onflow/flow-go/module/signature" ) // CombinedSignerV3 creates votes for the main consensus. @@ -22,12 +24,15 @@ import ( // The difference between V2 and V3 is that V2 will sign 2 sigs, whereas // V3 only sign 1 sig. type CombinedSignerV3 struct { - staking module.Local - stakingHasher hash.Hasher - beaconKeyStore module.RandomBeaconKeyStore - beaconHasher hash.Hasher + staking module.Local + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher + beaconKeyStore module.RandomBeaconKeyStore + beaconHasher hash.Hasher } +var _ hotstuff.Signer = (*CombinedSignerV3)(nil) + // NewCombinedSignerV3 creates a new combined signer with the given dependencies: // - the staking signer is used to create and verify aggregatable signatures for Hotstuff // - the beaconKeyStore is used to get threshold-signers by epoch/view; @@ -38,10 +43,11 @@ func NewCombinedSignerV3( ) *CombinedSignerV3 { sc := &CombinedSignerV3{ - staking: staking, - stakingHasher: signature.NewBLSHasher(signature.ConsensusVoteTag), - beaconKeyStore: beaconKeyStore, - beaconHasher: signature.NewBLSHasher(signature.RandomBeaconTag), + staking: staking, + stakingHasher: msig.NewBLSHasher(msig.ConsensusVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.ConsensusTimeoutTag), + beaconKeyStore: beaconKeyStore, + beaconHasher: msig.NewBLSHasher(msig.RandomBeaconTag), } return sc } @@ -89,6 +95,26 @@ func (c *CombinedSignerV3) CreateVote(block *model.Block) (*model.Vote, error) { return vote, nil } +// CreateTimeout will create a signed timeout object for the given view. +// Timeout objects are only signed with the staking key (not beacon key). +func (c *CombinedSignerV3) CreateTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + // create timeout object specific message + msg := MakeTimeoutMessage(curView, newestQC.View) + sigData, err := c.staking.Sign(msg, c.timeoutObjectHasher) + if err != nil { + return nil, fmt.Errorf("could not generate signature for timeout object at view %d: %w", curView, err) + } + + timeout := &model.TimeoutObject{ + View: curView, + NewestQC: newestQC, + LastViewTC: lastViewTC, + SignerID: c.staking.NodeID(), + SigData: sigData, + } + return timeout, nil +} + // genSigData generates the signature data for our local node for the given block. func (c *CombinedSignerV3) genSigData(block *model.Block) ([]byte, error) { @@ -105,7 +131,7 @@ func (c *CombinedSignerV3) genSigData(block *model.Block) ([]byte, error) { return nil, fmt.Errorf("could not generate staking signature: %w", err) } - return signature.EncodeSingleSig(encoding.SigTypeStaking, stakingSig), nil + return msig.EncodeSingleSig(encoding.SigTypeStaking, stakingSig), nil } return nil, fmt.Errorf("could not get random beacon private key for view %d: %w", block.View, err) } @@ -117,5 +143,5 @@ func (c *CombinedSignerV3) genSigData(block *model.Block) ([]byte, error) { return nil, fmt.Errorf("could not generate beacon signature: %w", err) } - return signature.EncodeSingleSig(encoding.SigTypeRandomBeacon, beaconShare), nil + return msig.EncodeSingleSig(encoding.SigTypeRandomBeacon, beaconShare), nil } diff --git a/consensus/hotstuff/verification/combined_signer_v3_test.go b/consensus/hotstuff/verification/combined_signer_v3_test.go index 614c4673840..90dc4ed1f09 100644 --- a/consensus/hotstuff/verification/combined_signer_v3_test.go +++ b/consensus/hotstuff/verification/combined_signer_v3_test.go @@ -32,7 +32,7 @@ func TestCombinedSignWithDKGKeyV3(t *testing.T) { fblock := unittest.BlockFixture() fblock.Header.View = view - block := model.BlockFromFlow(fblock.Header, 10) + block := model.BlockFromFlow(fblock.Header) signerID := fblock.Header.ProposerID epochCounter := uint64(3) @@ -57,7 +57,7 @@ func TestCombinedSignWithDKGKeyV3(t *testing.T) { dkg := &mocks.DKG{} dkg.On("KeyShare", signerID).Return(pk, nil) - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} committee.On("DKG", mock.Anything).Return(dkg, nil) packer := signature.NewConsensusSigDataPacker(committee) @@ -68,7 +68,7 @@ func TestCombinedSignWithDKGKeyV3(t *testing.T) { require.NoError(t, err) vote := proposal.ProposerVote() - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) // check that a created proposal's signature is a combined staking sig and random beacon sig @@ -85,7 +85,7 @@ func TestCombinedSignWithDKGKeyV3(t *testing.T) { // as a sign of an invalid vote and wraps it into a `model.InvalidSignerError`. *dkg = mocks.DKG{} // overwrite DKG mock with a new one dkg.On("KeyShare", signerID).Return(nil, protocol.IdentityNotFoundError{NodeID: signerID}) - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.True(t, model.IsInvalidSignerError(err)) } @@ -99,7 +99,7 @@ func TestCombinedSignWithNoDKGKeyV3(t *testing.T) { fblock := unittest.BlockFixture() fblock.Header.View = view - block := model.BlockFromFlow(fblock.Header, 10) + block := model.BlockFromFlow(fblock.Header) signerID := fblock.Header.ProposerID epochCounter := uint64(3) @@ -124,7 +124,7 @@ func TestCombinedSignWithNoDKGKeyV3(t *testing.T) { dkg := &mocks.DKG{} dkg.On("KeyShare", signerID).Return(pk, nil) - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} // even if the node failed DKG, and has no random beacon private key, // but other nodes, who completed and succeeded DKG, have a public key // for this failed node, which can be used to verify signature from @@ -138,7 +138,7 @@ func TestCombinedSignWithNoDKGKeyV3(t *testing.T) { require.NoError(t, err) vote := proposal.ProposerVote() - err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block) + err = verifier.VerifyVote(nodeID, vote.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) // check that a created proposal's signature is a combined staking sig and random beacon sig @@ -155,7 +155,7 @@ func TestCombinedSignWithNoDKGKeyV3(t *testing.T) { // Test_VerifyQC checks that a QC where either signer list is empty is rejected as invalid func Test_VerifyQCV3(t *testing.T) { header := unittest.BlockHeaderFixture() - block := model.BlockFromFlow(header, header.View-1) + block := model.BlockFromFlow(header) msg := MakeVoteMessage(block.View, block.BlockID) // generate some BLS key as a stub of the random beacon group key and use it to generate a reconstructed beacon sig @@ -163,7 +163,7 @@ func Test_VerifyQCV3(t *testing.T) { dkg := &mocks.DKG{} dkg.On("GroupKey").Return(privGroupKey.PublicKey(), nil) dkg.On("Size").Return(uint(20)) - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} committee.On("DKG", mock.Anything).Return(dkg, nil) // generate 17 BLS keys as stubs for staking keys and use them to generate an aggregated staking sig @@ -191,7 +191,7 @@ func Test_VerifyQCV3(t *testing.T) { packer.On("Unpack", mock.Anything, packedSigData).Return(&unpackedSigData, nil) verifier := NewCombinedVerifierV3(committee, packer) - err := verifier.VerifyQC(allSigners, packedSigData, block) + err := verifier.VerifyQC(allSigners, packedSigData, block.View, block.BlockID) require.NoError(t, err) }) @@ -208,7 +208,7 @@ func Test_VerifyQCV3(t *testing.T) { packer := &mocks.Packer{} packer.On("Unpack", mock.Anything, packedSigData).Return(&sd, nil) verifier := NewCombinedVerifierV3(committee, packer) - err := verifier.VerifyQC(allSigners, packedSigData, block) + err := verifier.VerifyQC(allSigners, packedSigData, block.View, block.BlockID) require.NoError(t, err) }) @@ -223,7 +223,7 @@ func Test_VerifyQCV3(t *testing.T) { packer := &mocks.Packer{} packer.On("Unpack", mock.Anything, packedSigData).Return(&sd, nil) verifier := NewCombinedVerifierV3(committee, packer) - err := verifier.VerifyQC(allSigners, packedSigData, block) + err := verifier.VerifyQC(allSigners, packedSigData, block.View, block.BlockID) require.True(t, model.IsInvalidFormatError(err)) }) @@ -236,7 +236,7 @@ func Test_VerifyQCV3(t *testing.T) { packer := &mocks.Packer{} packer.On("Unpack", mock.Anything, packedSigData).Return(&sd, nil) verifier := NewCombinedVerifierV3(committee, packer) - err := verifier.VerifyQC(allSigners, packedSigData, block) + err := verifier.VerifyQC(allSigners, packedSigData, block.View, block.BlockID) require.True(t, model.IsInvalidFormatError(err)) }) @@ -252,7 +252,7 @@ func Test_VerifyQCV3(t *testing.T) { packer := &mocks.Packer{} packer.On("Unpack", mock.Anything, packedSigData).Return(&sd, nil) verifier := NewCombinedVerifierV3(committee, packer) - err := verifier.VerifyQC(allSigners, packedSigData, block) + err := verifier.VerifyQC(allSigners, packedSigData, block.View, block.BlockID) require.True(t, model.IsInvalidFormatError(err)) }) @@ -263,18 +263,18 @@ func Test_VerifyQCV3(t *testing.T) { // any sub-components, because some (e.g. `crypto.AggregateBLSPublicKeys`) don't provide sufficient // sentinel errors to distinguish between internal problems and external byzantine inputs. func Test_VerifyQC_EmptySignersV3(t *testing.T) { - committee := &mocks.Committee{} + committee := &mocks.DynamicCommittee{} packer := signature.NewConsensusSigDataPacker(committee) verifier := NewCombinedVerifier(committee, packer) header := unittest.BlockHeaderFixture() - block := model.BlockFromFlow(header, header.View-1) + block := model.BlockFromFlow(header) sigData := unittest.QCSigDataFixture() - err := verifier.VerifyQC([]*flow.Identity{}, sigData, block) + err := verifier.VerifyQC([]*flow.Identity{}, sigData, block.View, block.BlockID) require.True(t, model.IsInsufficientSignaturesError(err)) - err = verifier.VerifyQC(nil, sigData, block) + err = verifier.VerifyQC(nil, sigData, block.View, block.BlockID) require.True(t, model.IsInsufficientSignaturesError(err)) } diff --git a/consensus/hotstuff/verification/combined_verifier_v2.go b/consensus/hotstuff/verification/combined_verifier_v2.go index 6b742a50276..1bfa4936e0e 100644 --- a/consensus/hotstuff/verification/combined_verifier_v2.go +++ b/consensus/hotstuff/verification/combined_verifier_v2.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/signature" + msig "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/state/protocol" ) @@ -22,10 +22,11 @@ import ( // a signature from a random beacon signer, which verifies either the signature share or // the reconstructed threshold signature. type CombinedVerifier struct { - committee hotstuff.Committee - stakingHasher hash.Hasher - beaconHasher hash.Hasher - packer hotstuff.Packer + committee hotstuff.Replicas + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher + beaconHasher hash.Hasher + packer hotstuff.Packer } var _ hotstuff.Verifier = (*CombinedVerifier)(nil) @@ -34,12 +35,13 @@ var _ hotstuff.Verifier = (*CombinedVerifier)(nil) // - the hotstuff committee's state is used to retrieve the public keys for the staking signature; // - the merger is used to combine and split staking and random beacon signatures; // - the packer is used to unpack QC for verification; -func NewCombinedVerifier(committee hotstuff.Committee, packer hotstuff.Packer) *CombinedVerifier { +func NewCombinedVerifier(committee hotstuff.Replicas, packer hotstuff.Packer) *CombinedVerifier { return &CombinedVerifier{ - committee: committee, - stakingHasher: signature.NewBLSHasher(signature.ConsensusVoteTag), - beaconHasher: signature.NewBLSHasher(signature.RandomBeaconTag), - packer: packer, + committee: committee, + stakingHasher: msig.NewBLSHasher(msig.ConsensusVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.ConsensusTimeoutTag), + beaconHasher: msig.NewBLSHasher(msig.RandomBeaconTag), + packer: packer, } } @@ -49,23 +51,24 @@ func NewCombinedVerifier(committee hotstuff.Committee, packer hotstuff.Packer) * // - model.InvalidFormatError if the signature has an incompatible format. // - model.ErrInvalidSignature is the signature is invalid // - model.InvalidSignerError if signer is _not_ part of the random beacon committee +// - model.ErrViewForUnknownEpoch if no epoch containing the given view is known // - unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) -func (c *CombinedVerifier) VerifyVote(signer *flow.Identity, sigData []byte, block *model.Block) error { +func (c *CombinedVerifier) VerifyVote(signer *flow.Identity, sigData []byte, view uint64, blockID flow.Identifier) error { // create the to-be-signed message - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) // split the two signatures from the vote - stakingSig, beaconShare, err := signature.DecodeDoubleSig(sigData) + stakingSig, beaconShare, err := msig.DecodeDoubleSig(sigData) if err != nil { - if errors.Is(err, signature.ErrInvalidSignatureFormat) { - return model.NewInvalidFormatErrorf("could not split signature for block %v: %w", block.BlockID, err) + if errors.Is(err, msig.ErrInvalidSignatureFormat) { + return model.NewInvalidFormatErrorf("could not split signature for block %v: %w", blockID, err) } - return fmt.Errorf("unexpected internal error while splitting signature for block %v: %w", block.BlockID, err) + return fmt.Errorf("unexpected internal error while splitting signature for block %v: %w", blockID, err) } - dkg, err := c.committee.DKG(block.BlockID) + dkg, err := c.committee.DKG(view) if err != nil { return fmt.Errorf("could not get dkg: %w", err) } @@ -75,10 +78,10 @@ func (c *CombinedVerifier) VerifyVote(signer *flow.Identity, sigData []byte, blo stakingValid, err := signer.StakingPubKey.Verify(stakingSig, msg, c.stakingHasher) if err != nil { return fmt.Errorf("internal error while verifying staking signature of node %x at block %v: %w", - signer.NodeID, block.BlockID, err) + signer.NodeID, blockID, err) } if !stakingValid { - return fmt.Errorf("invalid staking sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid staking sig for block %v: %w", blockID, model.ErrInvalidSignature) } // there is no beacon share, no need to verify it @@ -93,16 +96,16 @@ func (c *CombinedVerifier) VerifyVote(signer *flow.Identity, sigData []byte, blo return model.NewInvalidSignerErrorf("%v is not a random beacon participant: %w", signer.NodeID, err) } return fmt.Errorf("unexpected error retrieving random beacon key share for node %x at block %v: %w", - signer.NodeID, block.BlockID, err) + signer.NodeID, blockID, err) } beaconValid, err := beaconPubKey.Verify(beaconShare, msg, c.beaconHasher) if err != nil { return fmt.Errorf("internal error while verifying beacon signature at block %v: %w", - block.BlockID, err) + blockID, err) } if !beaconValid { - return fmt.Errorf("invalid beacon sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid beacon sig for block %v: %w", blockID, model.ErrInvalidSignature) } return nil } @@ -113,15 +116,16 @@ func (c *CombinedVerifier) VerifyVote(signer *flow.Identity, sigData []byte, blo // - nil if `sigData` is cryptographically valid // - model.InsufficientSignaturesError if `signers` is empty. // Depending on the order of checks in the higher-level logic this error might -// be an indicator of a external byzantine input or an internal bug. +// be an indicator of an external byzantine input or an internal bug. // - model.InvalidFormatError if `sigData` has an incompatible format // - model.ErrInvalidSignature if a signature is invalid +// - model.ErrViewForUnknownEpoch if no epoch containing the given view is known // - error if running into any unexpected exception (i.e. fatal error) -func (c *CombinedVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, block *model.Block) error { +func (c *CombinedVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, view uint64, blockID flow.Identifier) error { if len(signers) == 0 { return model.NewInsufficientSignaturesErrorf("empty list of signers") } - dkg, err := c.committee.DKG(block.BlockID) + dkg, err := c.committee.DKG(view) if err != nil { return fmt.Errorf("could not get dkg data: %w", err) } @@ -132,7 +136,7 @@ func (c *CombinedVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, b return fmt.Errorf("could not split signature: %w", err) } - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) // verify the beacon signature first since it is faster to verify (no public key aggregation needed) beaconValid, err := dkg.GroupKey().Verify(blockSigData.ReconstructedRandomBeaconSig, msg, c.beaconHasher) @@ -140,7 +144,7 @@ func (c *CombinedVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, b return fmt.Errorf("internal error while verifying beacon signature: %w", err) } if !beaconValid { - return fmt.Errorf("invalid reconstructed random beacon sig for block (%x): %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid reconstructed random beacon sig for block (%x): %w", blockID, model.ErrInvalidSignature) } // aggregate public staking keys of all signers (more costly) @@ -158,17 +162,30 @@ func (c *CombinedVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, b // By checking `len(signers) == 0` upfront , we can rule out case (i) as a source of error. // Hence, if we encounter an error here, we know it is case (ii). Thereby, we can clearly // distinguish a faulty _external_ input from an _internal_ uncovered edge-case. - return fmt.Errorf("could not compute aggregated key for block %x: %w", block.BlockID, err) + return fmt.Errorf("could not compute aggregated key for block %x: %w", blockID, err) } // verify aggregated signature with aggregated keys from last step stakingValid, err := aggregatedKey.Verify(blockSigData.AggregatedStakingSig, msg, c.stakingHasher) if err != nil { - return fmt.Errorf("internal error while verifying staking signature for block %x: %w", block.BlockID, err) + return fmt.Errorf("internal error while verifying staking signature for block %x: %w", blockID, err) } if !stakingValid { - return fmt.Errorf("invalid aggregated staking sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid aggregated staking sig for block %v: %w", blockID, model.ErrInvalidSignature) } return nil } + +// VerifyTC checks cryptographic validity of the TC's `sigData` w.r.t. the +// given view. It is the responsibility of the calling code to ensure +// that all `signers` are authorized, without duplicates. Return values: +// - nil if `sigData` is cryptographically valid +// - model.InsufficientSignaturesError if `signers is empty. +// - model.InvalidFormatError if `signers`/`highQCViews` have differing lengths +// - model.ErrInvalidSignature if a signature is invalid +// - unexpected errors should be treated as symptoms of bugs or uncovered +// edge cases in the logic (i.e. as fatal) +func (c *CombinedVerifier) VerifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64) error { + return verifyTC(signers, sigData, view, highQCViews, c.timeoutObjectHasher) +} diff --git a/consensus/hotstuff/verification/combined_verifier_v3.go b/consensus/hotstuff/verification/combined_verifier_v3.go index 534ccaa526e..fc7e2c582c5 100644 --- a/consensus/hotstuff/verification/combined_verifier_v3.go +++ b/consensus/hotstuff/verification/combined_verifier_v3.go @@ -23,10 +23,11 @@ import ( // a signature from a random beacon signer, which verifies both the signature share and // the reconstructed threshold signature. type CombinedVerifierV3 struct { - committee hotstuff.Committee - stakingHasher hash.Hasher - beaconHasher hash.Hasher - packer hotstuff.Packer + committee hotstuff.Replicas + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher + beaconHasher hash.Hasher + packer hotstuff.Packer } var _ hotstuff.Verifier = (*CombinedVerifierV3)(nil) @@ -34,12 +35,13 @@ var _ hotstuff.Verifier = (*CombinedVerifierV3)(nil) // NewCombinedVerifierV3 creates a new combined verifier with the given dependencies. // - the hotstuff committee's state is used to retrieve the public keys for the staking signature; // - the packer is used to unpack QC for verification; -func NewCombinedVerifierV3(committee hotstuff.Committee, packer hotstuff.Packer) *CombinedVerifierV3 { +func NewCombinedVerifierV3(committee hotstuff.Replicas, packer hotstuff.Packer) *CombinedVerifierV3 { return &CombinedVerifierV3{ - committee: committee, - stakingHasher: msig.NewBLSHasher(msig.ConsensusVoteTag), - beaconHasher: msig.NewBLSHasher(msig.RandomBeaconTag), - packer: packer, + committee: committee, + stakingHasher: msig.NewBLSHasher(msig.ConsensusVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.ConsensusTimeoutTag), + beaconHasher: msig.NewBLSHasher(msig.RandomBeaconTag), + packer: packer, } } @@ -49,22 +51,23 @@ func NewCombinedVerifierV3(committee hotstuff.Committee, packer hotstuff.Packer) // - model.InvalidFormatError if the signature has an incompatible format. // - model.ErrInvalidSignature is the signature is invalid // - model.InvalidSignerError if signer is _not_ part of the random beacon committee +// - model.ErrViewForUnknownEpoch if no epoch containing the given view is known // - unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) // // This implementation already support the cases, where the DKG committee is a // _strict subset_ of the full consensus committee. -func (c *CombinedVerifierV3) VerifyVote(signer *flow.Identity, sigData []byte, block *model.Block) error { +func (c *CombinedVerifierV3) VerifyVote(signer *flow.Identity, sigData []byte, view uint64, blockID flow.Identifier) error { // create the to-be-signed message - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) sigType, sig, err := msig.DecodeSingleSig(sigData) if err != nil { if errors.Is(err, msig.ErrInvalidSignatureFormat) { - return model.NewInvalidFormatErrorf("could not decode signature for block %v: %w", block.BlockID, err) + return model.NewInvalidFormatErrorf("could not decode signature for block %v: %w", blockID, err) } - return fmt.Errorf("unexpected internal error while decoding signature for block %v: %w", block.BlockID, err) + return fmt.Errorf("unexpected internal error while decoding signature for block %v: %w", blockID, err) } switch sigType { @@ -72,14 +75,14 @@ func (c *CombinedVerifierV3) VerifyVote(signer *flow.Identity, sigData []byte, b // verify each signature against the message stakingValid, err := signer.StakingPubKey.Verify(sig, msg, c.stakingHasher) if err != nil { - return fmt.Errorf("internal error while verifying staking signature for block %v: %w", block.BlockID, err) + return fmt.Errorf("internal error while verifying staking signature for block %v: %w", blockID, err) } if !stakingValid { - return fmt.Errorf("invalid staking sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid staking sig for block %v: %w", blockID, model.ErrInvalidSignature) } case encoding.SigTypeRandomBeacon: - dkg, err := c.committee.DKG(block.BlockID) + dkg, err := c.committee.DKG(view) if err != nil { return fmt.Errorf("could not get dkg: %w", err) } @@ -90,14 +93,14 @@ func (c *CombinedVerifierV3) VerifyVote(signer *flow.Identity, sigData []byte, b if protocol.IsIdentityNotFound(err) { return model.NewInvalidSignerErrorf("%v is not a random beacon participant: %w", signer.NodeID, err) } - return fmt.Errorf("could not get random beacon key share for %x at block %v: %w", signer.NodeID, block.BlockID, err) + return fmt.Errorf("could not get random beacon key share for %x at block %v: %w", signer.NodeID, blockID, err) } beaconValid, err := beaconPubKey.Verify(sig, msg, c.beaconHasher) if err != nil { - return fmt.Errorf("internal error while verifying beacon signature for block %v: %w", block.BlockID, err) + return fmt.Errorf("internal error while verifying beacon signature for block %v: %w", blockID, err) } if !beaconValid { - return fmt.Errorf("invalid beacon sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid beacon sig for block %v: %w", blockID, model.ErrInvalidSignature) } default: @@ -117,16 +120,17 @@ func (c *CombinedVerifierV3) VerifyVote(signer *flow.Identity, sigData []byte, b // - model.InvalidFormatError if `sigData` has an incompatible format // - model.ErrInvalidSignature if a signature is invalid // - model.InvalidSignerError if a signer is _not_ part of the random beacon committee +// - model.ErrViewForUnknownEpoch if no epoch containing the given view is known // - error if running into any unexpected exception (i.e. fatal error) // // This implementation already support the cases, where the DKG committee is a // _strict subset_ of the full consensus committee. -func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, block *model.Block) error { +func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, view uint64, blockID flow.Identifier) error { if len(signers) == 0 { return model.NewInsufficientSignaturesErrorf("empty list of signers") } signerIdentities := signers.Lookup() - dkg, err := c.committee.DKG(block.BlockID) + dkg, err := c.committee.DKG(view) if err != nil { return fmt.Errorf("could not get dkg data: %w", err) } @@ -137,7 +141,7 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, return fmt.Errorf("could not split signature: %w", err) } - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) // STEP 1: verify random beacon group key // We do this first, since it is faster to check (no public key aggregation needed). @@ -146,7 +150,7 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, return fmt.Errorf("internal error while verifying beacon signature: %w", err) } if !beaconValid { - return fmt.Errorf("invalid reconstructed random beacon sig for block (%x): %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid reconstructed random beacon sig for block (%x): %w", blockID, model.ErrInvalidSignature) } // verify the aggregated staking and beacon signatures next (more costly) @@ -162,7 +166,7 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, return fmt.Errorf("internal error while verifying aggregated signature: %w", err) } if !valid { - return fmt.Errorf("invalid aggregated sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid aggregated sig for block %v: %w", blockID, model.ErrInvalidSignature) } return nil } @@ -202,7 +206,7 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, // Our previous threshold check also guarantees that `beaconPubKeys` is not empty. err = verifyAggregatedSignature(beaconPubKeys, blockSigData.AggregatedRandomBeaconSig, c.beaconHasher) if err != nil { - return fmt.Errorf("verifying aggregated random beacon sig shares failed for block %v: %w", block.BlockID, err) + return fmt.Errorf("verifying aggregated random beacon sig shares failed for block %v: %w", blockID, err) } // STEP 3: validating the aggregated staking signatures @@ -212,7 +216,7 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, numStakingSigners := len(blockSigData.StakingSigners) if numStakingSigners == 0 { if len(blockSigData.AggregatedStakingSig) > 0 { - return model.NewInvalidFormatErrorf("all replicas signed with random beacon keys, but QC has aggregated staking sig for block %v", block.BlockID) + return model.NewInvalidFormatErrorf("all replicas signed with random beacon keys, but QC has aggregated staking sig for block %v", blockID) } // no aggregated staking sig to verify return nil @@ -230,9 +234,22 @@ func (c *CombinedVerifierV3) VerifyQC(signers flow.IdentityList, sigData []byte, } err = verifyAggregatedSignature(stakingPubKeys, blockSigData.AggregatedStakingSig, c.stakingHasher) if err != nil { - return fmt.Errorf("verifying aggregated staking sig failed for block %v: %w", block.BlockID, err) + return fmt.Errorf("verifying aggregated staking sig failed for block %v: %w", blockID, err) } return nil } + +// VerifyTC checks cryptographic validity of the TC's `sigData` w.r.t. the +// given view. It is the responsibility of the calling code to ensure +// that all `signers` are authorized, without duplicates. Return values: +// - nil if `sigData` is cryptographically valid +// - model.InsufficientSignaturesError if `signers is empty. +// - model.InvalidFormatError if `signers`/`highQCViews` have differing lengths +// - model.ErrInvalidSignature if a signature is invalid +// - unexpected errors should be treated as symptoms of bugs or uncovered +// edge cases in the logic (i.e. as fatal) +func (c *CombinedVerifierV3) VerifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64) error { + return verifyTC(signers, sigData, view, highQCViews, c.timeoutObjectHasher) +} diff --git a/consensus/hotstuff/verification/common.go b/consensus/hotstuff/verification/common.go index e3adfc4382b..f3140b4656a 100644 --- a/consensus/hotstuff/verification/common.go +++ b/consensus/hotstuff/verification/common.go @@ -2,6 +2,8 @@ package verification import ( "github.com/onflow/flow-go/model/flow" + + "encoding/binary" ) // MakeVoteMessage generates the message we have to sign in order to be able @@ -10,12 +12,18 @@ import ( // block ID; this allows us to create the signed message and verify the signed // message without having the full block contents. func MakeVoteMessage(view uint64, blockID flow.Identifier) []byte { - msg := flow.MakeID(struct { - BlockID flow.Identifier - View uint64 - }{ - BlockID: blockID, - View: view, - }) - return msg[:] + msg := make([]byte, 8, 8+flow.IdentifierLen) + binary.BigEndian.PutUint64(msg, view) + msg = append(msg, blockID[:]...) + return msg +} + +// MakeTimeoutMessage generates the message we have to sign in order to be able +// to contribute to Active Pacemaker protocol. Each replica signs with the highest QC view +// known to that replica. +func MakeTimeoutMessage(view uint64, newestQCView uint64) []byte { + msg := make([]byte, 16) + binary.BigEndian.PutUint64(msg[:8], view) + binary.BigEndian.PutUint64(msg[8:], newestQCView) + return msg } diff --git a/consensus/hotstuff/verification/metrics_wrapper.go b/consensus/hotstuff/verification/metrics_wrapper.go index 1f0598d99d9..7c929e361ef 100644 --- a/consensus/hotstuff/verification/metrics_wrapper.go +++ b/consensus/hotstuff/verification/metrics_wrapper.go @@ -5,6 +5,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) @@ -19,6 +20,8 @@ type SignerMetricsWrapper struct { metrics module.HotstuffMetrics } +var _ hotstuff.Signer = (*SignerMetricsWrapper)(nil) + func NewMetricsWrapper(signer hotstuff.Signer, metrics module.HotstuffMetrics) *SignerMetricsWrapper { return &SignerMetricsWrapper{ signer: signer, @@ -55,6 +58,15 @@ func (w SignerMetricsWrapper) CreateVote(block *model.Block) (*model.Vote, error return vote, err } +func (w SignerMetricsWrapper) CreateTimeout(curView uint64, + newestQC *flow.QuorumCertificate, + lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + processStart := time.Now() + timeout, err := w.signer.CreateTimeout(curView, newestQC, lastViewTC) + w.metrics.SignerProcessingDuration(time.Since(processStart)) + return timeout, err +} + // func (w SignerMetricsWrapper) CreateQC(votes []*model.Vote) (*flow.QuorumCertificate, error) { // processStart := time.Now() // qc, err := w.signer.CreateQC(votes) diff --git a/consensus/hotstuff/verification/staking_signer.go b/consensus/hotstuff/verification/staking_signer.go index a037ea06903..91c35e1cddd 100644 --- a/consensus/hotstuff/verification/staking_signer.go +++ b/consensus/hotstuff/verification/staking_signer.go @@ -3,6 +3,7 @@ package verification import ( "fmt" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/model/flow" @@ -15,11 +16,14 @@ import ( // as part of their vote. StakingSigner is responsible for creating correctly // signed proposals and votes. type StakingSigner struct { - me module.Local - stakingHasher hash.Hasher - signerID flow.Identifier + me module.Local + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher + signerID flow.Identifier } +var _ hotstuff.Signer = (*StakingSigner)(nil) + // NewStakingSigner instantiates a StakingSigner, which signs votes and // proposals with the staking key. The generated signatures are aggregatable. func NewStakingSigner( @@ -27,9 +31,10 @@ func NewStakingSigner( ) *StakingSigner { sc := &StakingSigner{ - me: me, - stakingHasher: msig.NewBLSHasher(msig.CollectorVoteTag), - signerID: me.NodeID(), + me: me, + stakingHasher: msig.NewBLSHasher(msig.CollectorVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.CollectorTimeoutTag), + signerID: me.NodeID(), } return sc } @@ -77,6 +82,25 @@ func (c *StakingSigner) CreateVote(block *model.Block) (*model.Vote, error) { return vote, nil } +// CreateTimeout will create a signed timeout object for the given view. +func (c *StakingSigner) CreateTimeout(curView uint64, newestQC *flow.QuorumCertificate, lastViewTC *flow.TimeoutCertificate) (*model.TimeoutObject, error) { + // create timeout object specific message + msg := MakeTimeoutMessage(curView, newestQC.View) + sigData, err := c.me.Sign(msg, c.timeoutObjectHasher) + if err != nil { + return nil, fmt.Errorf("could not generate signature for timeout object at view %d: %w", curView, err) + } + + timeout := &model.TimeoutObject{ + View: curView, + NewestQC: newestQC, + LastViewTC: lastViewTC, + SignerID: c.signerID, + SigData: sigData, + } + return timeout, nil +} + // genSigData generates the signature data for our local node for the given block. // It returns: // - (stakingSig, nil) signature signed with staking key. The sig is 48 bytes long diff --git a/consensus/hotstuff/verification/staking_signer_test.go b/consensus/hotstuff/verification/staking_signer_test.go index 9122d0f067c..fc563266f92 100644 --- a/consensus/hotstuff/verification/staking_signer_test.go +++ b/consensus/hotstuff/verification/staking_signer_test.go @@ -61,7 +61,7 @@ func TestStakingSigner_CreateProposal(t *testing.T) { require.NotNil(t, proposal) verifier := NewStakingVerifier() - err = verifier.VerifyVote(signerIdentity, proposal.SigData, proposal.Block) + err = verifier.VerifyVote(signerIdentity, proposal.SigData, proposal.Block.View, proposal.Block.BlockID) require.NoError(t, err) }) } @@ -102,7 +102,7 @@ func TestStakingSigner_CreateVote(t *testing.T) { require.NotNil(t, vote) verifier := NewStakingVerifier() - err = verifier.VerifyVote(signerIdentity, vote.SigData, block) + err = verifier.VerifyVote(signerIdentity, vote.SigData, block.View, block.BlockID) require.NoError(t, err) }) } @@ -110,13 +110,13 @@ func TestStakingSigner_CreateVote(t *testing.T) { // TestStakingSigner_VerifyQC checks that a QC without any signers is rejected right away without calling into any sub-components func TestStakingSigner_VerifyQC(t *testing.T) { header := unittest.BlockHeaderFixture() - block := model.BlockFromFlow(header, header.View-1) + block := model.BlockFromFlow(header) sigData := unittest.RandomBytes(127) verifier := NewStakingVerifier() - err := verifier.VerifyQC([]*flow.Identity{}, sigData, block) - require.True(t, model.IsInvalidFormatError(err)) + err := verifier.VerifyQC([]*flow.Identity{}, sigData, block.View, block.BlockID) + require.True(t, model.IsInsufficientSignaturesError(err)) - err = verifier.VerifyQC(nil, sigData, block) - require.True(t, model.IsInvalidFormatError(err)) + err = verifier.VerifyQC(nil, sigData, block.View, block.BlockID) + require.True(t, model.IsInsufficientSignaturesError(err)) } diff --git a/consensus/hotstuff/verification/staking_verifier.go b/consensus/hotstuff/verification/staking_verifier.go index 283ce27424a..fd83d3b318e 100644 --- a/consensus/hotstuff/verification/staking_verifier.go +++ b/consensus/hotstuff/verification/staking_verifier.go @@ -17,7 +17,8 @@ import ( // StakingVerifier is a verifier capable of verifying staking signature for each // verifying operation. It's used primarily with collection cluster where hotstuff without beacon signers is used. type StakingVerifier struct { - stakingHasher hash.Hasher + stakingHasher hash.Hasher + timeoutObjectHasher hash.Hasher } var _ hotstuff.Verifier = (*StakingVerifier)(nil) @@ -25,7 +26,8 @@ var _ hotstuff.Verifier = (*StakingVerifier)(nil) // NewStakingVerifier creates a new single verifier with the given dependencies. func NewStakingVerifier() *StakingVerifier { return &StakingVerifier{ - stakingHasher: msig.NewBLSHasher(msig.CollectorVoteTag), + stakingHasher: msig.NewBLSHasher(msig.CollectorVoteTag), + timeoutObjectHasher: msig.NewBLSHasher(msig.CollectorTimeoutTag), } } @@ -33,14 +35,13 @@ func NewStakingVerifier() *StakingVerifier { // Usually this method is only used to verify the proposer's vote, which is // the vote included in a block proposal. // The implementation returns the following sentinel errors: -// - model.InvalidFormatError if the signature has an incompatible format. // - model.ErrInvalidSignature is the signature is invalid // - unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) -func (v *StakingVerifier) VerifyVote(signer *flow.Identity, sigData []byte, block *model.Block) error { +func (v *StakingVerifier) VerifyVote(signer *flow.Identity, sigData []byte, view uint64, blockID flow.Identifier) error { // create the to-be-signed message - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) // verify each signature against the message stakingValid, err := signer.StakingPubKey.Verify(sigData, msg, v.stakingHasher) @@ -48,7 +49,7 @@ func (v *StakingVerifier) VerifyVote(signer *flow.Identity, sigData []byte, bloc return fmt.Errorf("internal error while verifying staking signature: %w", err) } if !stakingValid { - return fmt.Errorf("invalid sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid sig for block %v: %w", blockID, model.ErrInvalidSignature) } return nil @@ -56,19 +57,18 @@ func (v *StakingVerifier) VerifyVote(signer *flow.Identity, sigData []byte, bloc // VerifyQC checks the cryptographic validity of the QC's `sigData` for the // given block. It is the responsibility of the calling code to ensure -// that all `voters` are authorized, without duplicates. Return values: +// that all `signers` are authorized, without duplicates. Return values: // - nil if `sigData` is cryptographically valid -// - model.InvalidFormatError if `sigData` has an incompatible format // - model.ErrInvalidSignature if a signature is invalid // - unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) // // In the single verification case, `sigData` represents a single signature (`crypto.Signature`). -func (v *StakingVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, block *model.Block) error { +func (v *StakingVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, view uint64, blockID flow.Identifier) error { if len(signers) == 0 { - return model.NewInvalidFormatErrorf("empty list of signers") + return model.NewInsufficientSignaturesErrorf("empty list of signers") } - msg := MakeVoteMessage(block.View, block.BlockID) + msg := MakeVoteMessage(view, blockID) // verify the aggregated staking signature // TODO: to be replaced by module/signature.PublicKeyAggregator in V2 @@ -93,7 +93,56 @@ func (v *StakingVerifier) VerifyQC(signers flow.IdentityList, sigData []byte, bl } if !stakingValid { - return fmt.Errorf("invalid aggregated staking sig for block %v: %w", block.BlockID, model.ErrInvalidSignature) + return fmt.Errorf("invalid aggregated staking sig for block %v: %w", blockID, model.ErrInvalidSignature) + } + return nil +} + +// VerifyTC checks cryptographic validity of the TC's `sigData` w.r.t. the +// given view. It is the responsibility of the calling code to ensure +// that all `signers` are authorized, without duplicates. Return values: +// - nil if `sigData` is cryptographically valid +// - model.InsufficientSignaturesError if `signers is empty. +// - model.InvalidFormatError if `signers`/`highQCViews` have differing lengths +// - model.ErrInvalidSignature if a signature is invalid +// - unexpected errors should be treated as symptoms of bugs or uncovered +// edge cases in the logic (i.e. as fatal) +func (v *StakingVerifier) VerifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64) error { + return verifyTC(signers, sigData, view, highQCViews, v.timeoutObjectHasher) +} + +// verifyTC checks cryptographic validity of the TC's `sigData` w.r.t. the +// given view. It is the responsibility of the calling code to ensure +// that all `signers` are authorized, without duplicates. Return values: +// - nil if `sigData` is cryptographically valid +// - model.InsufficientSignaturesError if `signers is empty. +// - model.InvalidFormatError if `signers`/`highQCViews` have differing lengths +// - model.ErrInvalidSignature if a signature is invalid +// - unexpected errors should be treated as symptoms of bugs or uncovered +// edge cases in the logic (i.e. as fatal) +func verifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64, hasher hash.Hasher) error { + if len(signers) == 0 { + return model.NewInsufficientSignaturesErrorf("empty list of signers") + } + if len(signers) != len(highQCViews) { + return model.NewInvalidFormatErrorf("signers and highQCViews mismatch") + } + + pks := make([]crypto.PublicKey, 0, len(signers)) + messages := make([][]byte, 0, len(signers)) + hashers := make([]hash.Hasher, 0, len(signers)) + for i, identity := range signers { + pks = append(pks, identity.StakingPubKey) + messages = append(messages, MakeTimeoutMessage(view, highQCViews[i])) + hashers = append(hashers, hasher) + } + + valid, err := crypto.VerifyBLSSignatureManyMessages(pks, sigData, messages, hashers) + if err != nil { + return fmt.Errorf("signature verification failed: %w", err) + } + if !valid { + return fmt.Errorf("invalid aggregated TC signature for view %d: %w", view, model.ErrInvalidSignature) } return nil } diff --git a/consensus/hotstuff/verifier.go b/consensus/hotstuff/verifier.go index 8f2c5e5fa69..126ac7f78db 100644 --- a/consensus/hotstuff/verifier.go +++ b/consensus/hotstuff/verifier.go @@ -1,7 +1,6 @@ package hotstuff import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) @@ -9,14 +8,15 @@ import ( // votes, proposals and QC's against the block they are signing. // Overall, there are two criteria for the validity of a vote and QC: // -// (1) the signer ID(s) must correspond to authorized consensus participants -// (2) the signature must be cryptographically valid. +// (1) the signer ID(s) must correspond to authorized consensus participants +// (2) the signature must be cryptographically valid. // // Note that Verifier only implements (2). This API design allows to decouple -// (i) the common logic for checking that a super-majority of the consensus -// committee voted -// (ii) the handling of combined staking+RandomBeacon votes (consensus nodes) -// vs only staking votes (collector nodes) +// +// (i) the common logic for checking that a super-majority of the consensus +// committee voted +// (ii) the handling of combined staking+RandomBeacon votes (consensus nodes) +// vs only staking votes (collector nodes) // // On the one hand, this API design makes code less concise, as the two checks // are now distributed over API boundaries. On the other hand, we can avoid @@ -24,7 +24,7 @@ import ( type Verifier interface { // VerifyVote checks the cryptographic validity of a vote's `SigData` w.r.t. - // the given block. It is the responsibility of the calling code to ensure + // the view and blockID. It is the responsibility of the calling code to ensure // that `voter` is authorized to vote. // Return values: // * nil if `sigData` is cryptographically valid @@ -34,17 +34,19 @@ type Verifier interface { // where special signing authority is only given to a _subset_ of consensus // participants (e.g. random beacon). In case a participant signed despite not // being authorized, an InvalidSignerError is returned. + // * model.ErrViewForUnknownEpoch is only relevant for extended signature schemes, + // where querying of DKG might fail if no epoch containing the given view is known. // * unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) - VerifyVote(voter *flow.Identity, sigData []byte, block *model.Block) error + VerifyVote(voter *flow.Identity, sigData []byte, view uint64, blockID flow.Identifier) error // VerifyQC checks the cryptographic validity of a QC's `SigData` w.r.t. the - // given block. It is the responsibility of the calling code to ensure that + // given view and blockID. It is the responsibility of the calling code to ensure that // all `signers` are authorized, without duplicates. // Return values: // * nil if `sigData` is cryptographically valid // * model.InvalidFormatError if `sigData` has an incompatible format - // * model.InsufficientSignaturesError if `signers` is empty. + // * model.InsufficientSignaturesError if `signers is empty. // Depending on the order of checks in the higher-level logic this error might // be an indicator of a external byzantine input or an internal bug. // * model.ErrInvalidSignature if a signature is invalid @@ -52,7 +54,20 @@ type Verifier interface { // where special signing authority is only given to a _subset_ of consensus // participants (e.g. random beacon). In case a participant signed despite not // being authorized, an InvalidSignerError is returned. + // * model.ErrViewForUnknownEpoch is only relevant for extended signature schemes, + // where querying of DKG might fail if no epoch containing the given view is known. + // * unexpected errors should be treated as symptoms of bugs or uncovered + // edge cases in the logic (i.e. as fatal) + VerifyQC(signers flow.IdentityList, sigData []byte, view uint64, blockID flow.Identifier) error + + // VerifyTC checks cryptographic validity of the TC's `sigData` w.r.t. the + // given view. It is the responsibility of the calling code to ensure + // that all `signers` are authorized, without duplicates. Return values: + // * nil if `sigData` is cryptographically valid + // * model.InsufficientSignaturesError if `signers is empty. + // * model.InvalidFormatError if `signers`/`highQCViews` have differing lengths + // * model.ErrInvalidSignature if a signature is invalid // * unexpected errors should be treated as symptoms of bugs or uncovered // edge cases in the logic (i.e. as fatal) - VerifyQC(signers flow.IdentityList, sigData []byte, block *model.Block) error + VerifyTC(signers flow.IdentityList, sigData []byte, view uint64, highQCViews []uint64) error } diff --git a/consensus/hotstuff/vote_aggregator.go b/consensus/hotstuff/vote_aggregator.go index 588e176720a..14dc4f7dc2f 100644 --- a/consensus/hotstuff/vote_aggregator.go +++ b/consensus/hotstuff/vote_aggregator.go @@ -5,23 +5,6 @@ import ( "github.com/onflow/flow-go/module" ) -// BlockSigner abstracts the implementation of how a signature of a block or a vote is produced -// and stored in a stateful crypto object for aggregation. -// The VoteAggregator implements both the VoteAggregator interface and the BlockSigner interface so that -// the EventHandler could use the VoteAggregator interface to sign a Block, and Voter/BlockProducer can use -// the BlockSigner interface to create vote. -// When `CreateVote` is called, it internally creates stateful VoteCollector object, which also has the ability -// to verify the block and generate the vote signature. -// The created vote collector will be added to the vote collectors map. These -// implementation details are abstracted to Voter/BlockProducer. -type BlockSigner interface { - // CreateVote returns a vote for the given block. - // It returns: - // - (vote, nil) if vote is created - // - (nil , module.InvalidBlockError) if the block is invalid. - CreateVote(*model.Block) (*model.Vote, error) -} - // VoteAggregator verifies and aggregates votes to build QC. // When enough votes have been collected, it builds a QC and send it to the EventLoop // VoteAggregator also detects protocol violation, including invalid votes, double voting etc, and @@ -37,17 +20,16 @@ type VoteAggregator interface { AddVote(vote *model.Vote) // AddBlock notifies the VoteAggregator that it should start processing votes for the given block. - // AddBlock is a _synchronous_ call (logic is executed by the calling go routine). It also verifies - // validity of the proposer's vote for its own block. - // Expected error returns during normal operations: - // * model.InvalidBlockError if the proposer's vote for its own block is invalid - // * mempool.DecreasingPruningHeightError if the block's view has already been pruned - AddBlock(block *model.Proposal) error + // The input block is queued internally within the `VoteAggregator` and processed _asynchronously_ + // by the VoteAggregator's internal worker routines. + // CAUTION: we expect that the input block's validity has been confirmed prior to calling AddBlock, + // including the proposer's signature. Otherwise, VoteAggregator might crash or exhibit undefined + // behaviour. + AddBlock(block *model.Proposal) // InvalidBlock notifies the VoteAggregator about an invalid proposal, so that it - // can process votes for the invalid block and slash the voters. Expected error - // returns during normal operations: - // * mempool.DecreasingPruningHeightError if proposal's view has already been pruned + // can process votes for the invalid block and slash the voters. + // No errors are expected during normal operations InvalidBlock(block *model.Proposal) error // PruneUpToView deletes all votes _below_ to the given view, as well as diff --git a/consensus/hotstuff/vote_collector.go b/consensus/hotstuff/vote_collector.go index 7778e13d303..be5c6460723 100644 --- a/consensus/hotstuff/vote_collector.go +++ b/consensus/hotstuff/vote_collector.go @@ -64,9 +64,9 @@ type VoteCollector interface { ProcessBlock(block *model.Proposal) error // AddVote adds a vote to the collector - // return error if the signature is invalid // When enough votes have been added to produce a QC, the QC will be created asynchronously, and // passed to EventLoop through a callback. + // No errors are expected during normal operations. AddVote(vote *model.Vote) error // RegisterVoteConsumer registers a VoteConsumer. Upon registration, the collector diff --git a/consensus/hotstuff/vote_collectors.go b/consensus/hotstuff/vote_collectors.go index 6487e0b71f7..bf5ea61156b 100644 --- a/consensus/hotstuff/vote_collectors.go +++ b/consensus/hotstuff/vote_collectors.go @@ -3,7 +3,7 @@ package hotstuff import "github.com/onflow/flow-go/module" // VoteCollectors is an interface which allows VoteAggregator to interact with collectors structured by -// view and blockID. +// view. // Implementations of this interface are responsible for state transitions of `VoteCollector`s and pruning of // stale and outdated collectors by view. type VoteCollectors interface { @@ -15,11 +15,11 @@ type VoteCollectors interface { // When creating a vote collector, the view will be used to get epoch by view, then create the random beacon // signer object by epoch, because epoch determines DKG, which determines random beacon committee. // It returns: - // - (collector, true, nil) if no collector can be found by the block ID, and a new collector was created. - // - (collector, false, nil) if the collector can be found by the block ID + // - (collector, true, nil) if no collector can be found by the view, and a new collector was created. + // - (collector, false, nil) if the collector can be found by the view // - (nil, false, error) if running into any exception creating the vote collector state machine // Expected error returns during normal operations: - // * mempool.DecreasingPruningHeightError + // * mempool.BelowPrunedThresholdError - in case view is lower than last pruned view GetOrCreateCollector(view uint64) (collector VoteCollector, created bool, err error) // PruneUpToView prunes the vote collectors with views _below_ the given value, i.e. diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator.go b/consensus/hotstuff/voteaggregator/vote_aggregator.go index a4059b95733..6f0063f0037 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/rs/zerolog" @@ -12,9 +13,11 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/engine/consensus/sealing/counters" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" + "github.com/onflow/flow-go/module/metrics" ) // defaultVoteAggregatorWorkers number of workers to dispatch events for vote aggregators @@ -23,19 +26,25 @@ const defaultVoteAggregatorWorkers = 8 // defaultVoteQueueCapacity maximum capacity of buffering unprocessed votes const defaultVoteQueueCapacity = 1000 +// defaultBlockQueueCapacity maximum capacity of buffering unprocessed blocks +const defaultBlockQueueCapacity = 1000 + // VoteAggregator stores the votes and aggregates them into a QC when enough votes have been collected // VoteAggregator is designed in a way that it can aggregate votes for collection & consensus clusters // that is why implementation relies on dependency injection. type VoteAggregator struct { *component.ComponentManager log zerolog.Logger + hotstuffMetrics module.HotstuffMetrics + engineMetrics module.EngineMetrics notifier hotstuff.Consumer lowestRetainedView counters.StrictMonotonousCounter // lowest view, for which we still process votes collectors hotstuff.VoteCollectors - queuedVotesNotifier engine.Notifier + queuedMessagesNotifier engine.Notifier finalizationEventsNotifier engine.Notifier finalizedView counters.StrictMonotonousCounter // cache the last finalized view to queue up the pruning work, and unblock the caller who's delivering the finalization event. queuedVotes *fifoqueue.FifoQueue + queuedBlocks *fifoqueue.FifoQueue } var _ hotstuff.VoteAggregator = (*VoteAggregator)(nil) @@ -46,24 +55,36 @@ var _ component.Component = (*VoteAggregator)(nil) // different voting formats of main Consensus vs Collector consensus. func NewVoteAggregator( log zerolog.Logger, + hotstuffMetrics module.HotstuffMetrics, + engineMetrics module.EngineMetrics, + mempoolMetrics module.MempoolMetrics, notifier hotstuff.Consumer, lowestRetainedView uint64, collectors hotstuff.VoteCollectors, ) (*VoteAggregator, error) { - queuedVotes, err := fifoqueue.NewFifoQueue(defaultVoteQueueCapacity) + queuedVotes, err := fifoqueue.NewFifoQueue(defaultVoteQueueCapacity, + fifoqueue.WithLengthObserver(func(len int) { mempoolMetrics.MempoolEntries(metrics.ResourceBlockVoteQueue, uint(len)) })) if err != nil { return nil, fmt.Errorf("could not initialize votes queue") } + queuedBlocks, err := fifoqueue.NewFifoQueue(defaultBlockQueueCapacity) // TODO metrics + if err != nil { + return nil, fmt.Errorf("could not initialize blocks queue") + } + aggregator := &VoteAggregator{ - log: log, + log: log.With().Str("component", "hotstuff.vote_aggregator").Logger(), + hotstuffMetrics: hotstuffMetrics, + engineMetrics: engineMetrics, notifier: notifier, lowestRetainedView: counters.NewMonotonousCounter(lowestRetainedView), finalizedView: counters.NewMonotonousCounter(lowestRetainedView), collectors: collectors, queuedVotes: queuedVotes, - queuedVotesNotifier: engine.NewNotifier(), + queuedBlocks: queuedBlocks, + queuedMessagesNotifier: engine.NewNotifier(), finalizationEventsNotifier: engine.NewNotifier(), } @@ -71,11 +92,11 @@ func NewVoteAggregator( componentBuilder := component.NewComponentManagerBuilder() var wg sync.WaitGroup wg.Add(defaultVoteAggregatorWorkers) - for i := 0; i < defaultVoteAggregatorWorkers; i++ { // manager for worker routines that process inbound votes + for i := 0; i < defaultVoteAggregatorWorkers; i++ { // manager for worker routines that process inbound messages componentBuilder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + defer wg.Done() ready() - aggregator.queuedVotesProcessingLoop(ctx) - wg.Done() + aggregator.queuedMessagesProcessingLoop(ctx) }) } componentBuilder.AddWorker(func(_ irrecoverable.SignalerContext, ready component.ReadyFunc) { @@ -106,23 +127,26 @@ func NewVoteAggregator( return aggregator, nil } -func (va *VoteAggregator) queuedVotesProcessingLoop(ctx irrecoverable.SignalerContext) { - notifier := va.queuedVotesNotifier.Channel() +func (va *VoteAggregator) queuedMessagesProcessingLoop(ctx irrecoverable.SignalerContext) { + notifier := va.queuedMessagesNotifier.Channel() for { select { case <-ctx.Done(): return case <-notifier: - err := va.processQueuedVoteEvents(ctx) + err := va.processQueuedMessages(ctx) if err != nil { - ctx.Throw(fmt.Errorf("internal error processing queued vote events: %w", err)) + ctx.Throw(fmt.Errorf("internal error processing queued messages: %w", err)) return } } } } -func (va *VoteAggregator) processQueuedVoteEvents(ctx context.Context) error { +// processQueuedMessages is a function which dispatches previously queued messages on worker thread +// This function is called whenever we have queued messages ready to be dispatched. +// No errors are expected during normal operations. +func (va *VoteAggregator) processQueuedMessages(ctx context.Context) error { for { select { case <-ctx.Done(): @@ -130,20 +154,30 @@ func (va *VoteAggregator) processQueuedVoteEvents(ctx context.Context) error { default: } - msg, ok := va.queuedVotes.Pop() + msg, ok := va.queuedBlocks.Pop() + if ok { + block := msg.(*model.Proposal) + err := va.processQueuedBlock(block) + if err != nil { + return fmt.Errorf("could not process pending block %v: %w", block.Block.BlockID, err) + } + + continue + } + + msg, ok = va.queuedVotes.Pop() if ok { vote := msg.(*model.Vote) + startTime := time.Now() err := va.processQueuedVote(vote) + // report duration of processing one vote + va.hotstuffMetrics.VoteProcessingDuration(time.Since(startTime)) + va.engineMetrics.MessageHandled(metrics.EngineVoteAggregator, metrics.MessageBlockVote) + if err != nil { - return fmt.Errorf("could not process pending vote %v: %w", vote.ID(), err) + return fmt.Errorf("could not process pending vote %v for block %v: %w", vote.ID(), vote.BlockID, err) } - va.log.Info(). - Uint64("view", vote.View). - Hex("block_id", vote.BlockID[:]). - Str("vote_id", vote.ID().String()). - Msg("vote has been processed successfully") - continue } @@ -157,73 +191,52 @@ func (va *VoteAggregator) processQueuedVoteEvents(ctx context.Context) error { // concurrent goroutines. func (va *VoteAggregator) processQueuedVote(vote *model.Vote) error { collector, created, err := va.collectors.GetOrCreateCollector(vote.View) - if created { - va.log.Info().Uint64("view", vote.View).Msg("vote collector is created by processing vote") - } - if err != nil { // ignore if our routine is outdated and some other one has pruned collectors - if mempool.IsDecreasingPruningHeightError(err) { + if mempool.IsBelowPrunedThresholdError(err) { return nil } return fmt.Errorf("could not get collector for view %d: %w", vote.View, err) } + if created { + va.log.Info().Uint64("view", vote.View).Msg("vote collector is created by processing vote") + } + err = collector.AddVote(vote) if err != nil { - if model.IsDoubleVoteError(err) { - doubleVoteErr := err.(model.DoubleVoteError) - va.notifier.OnDoubleVotingDetected(doubleVoteErr.FirstVote, doubleVoteErr.ConflictingVote) - return nil - } - return fmt.Errorf("could not process vote for view %d, blockID %v: %w", vote.View, vote.BlockID, err) } - return nil -} - -// AddVote checks if vote is stale and appends vote into processing queue -// actual vote processing will be called in other dispatching goroutine. -func (va *VoteAggregator) AddVote(vote *model.Vote) { - // drop stale votes - if vote.View < va.lowestRetainedView.Value() { - - va.log.Info(). - Uint64("block_view", vote.View). - Hex("block_id", vote.BlockID[:]). - Hex("voter", vote.SignerID[:]). - Str("vote_id", vote.ID().String()). - Msg("drop stale votes") - - return - } + va.log.Info(). + Uint64("view", vote.View). + Hex("block_id", vote.BlockID[:]). + Str("vote_id", vote.ID().String()). + Msg("vote has been processed successfully") - // It's ok to silently drop votes in case our processing pipeline is full. - // It means that we are probably catching up. - if ok := va.queuedVotes.Push(vote); ok { - va.queuedVotesNotifier.Notify() - } + return nil } -// AddBlock notifies the VoteAggregator about a known block so that it can start processing -// pending votes whose block was unknown. -// It also verifies the proposer vote of a block, and return whether the proposer signature is valid. -// Expected error returns during normal operations: -// * model.InvalidBlockError if the proposer's vote for its own block is invalid -// * mempool.DecreasingPruningHeightError if the block's view has already been pruned -func (va *VoteAggregator) AddBlock(block *model.Proposal) error { +// processQueuedBlock performs actual processing of queued block proposals, this method is called from multiple +// concurrent goroutines. +// CAUTION: we expect that the input block's validity has been confirmed prior to calling AddBlock, +// including the proposer's signature. Otherwise, VoteAggregator might crash or exhibit undefined +// behaviour. +// No errors are expected during normal operation. +func (va *VoteAggregator) processQueuedBlock(block *model.Proposal) error { // check if the block is for a view that has already been pruned (and is thus stale) if block.Block.View < va.lowestRetainedView.Value() { - return mempool.NewDecreasingPruningHeightErrorf("block proposal for view %d is stale, lowestRetainedView is %d", block.Block.View, va.lowestRetainedView.Value()) + return nil } collector, created, err := va.collectors.GetOrCreateCollector(block.Block.View) if err != nil { + if mempool.IsBelowPrunedThresholdError(err) { + return nil + } return fmt.Errorf("could not get or create collector for block %v: %w", block.Block.BlockID, err) } - if created { va.log.Info(). Uint64("view", block.Block.View). @@ -233,16 +246,66 @@ func (va *VoteAggregator) AddBlock(block *model.Proposal) error { err = collector.ProcessBlock(block) if err != nil { + if model.IsInvalidBlockError(err) { + // We are attempting process a block which is invalid + // This should never happen, because any component that feeds blocks into VoteAggregator + // needs to make sure that it's submitting for processing ONLY valid blocks. + return fmt.Errorf("received invalid block for processing %v at view %d", block.Block.BlockID, block.Block.View) + } return fmt.Errorf("could not process block: %v, %w", block.Block.BlockID, err) } + va.log.Info(). + Uint64("view", block.Block.View). + Hex("block_id", block.Block.BlockID[:]). + Msg("block has been processed successfully") + return nil } +// AddVote checks if vote is stale and appends vote into processing queue +// actual vote processing will be called in other dispatching goroutine. +func (va *VoteAggregator) AddVote(vote *model.Vote) { + log := va.log.With().Uint64("block_view", vote.View). + Hex("block_id", vote.BlockID[:]). + Hex("voter", vote.SignerID[:]). + Str("vote_id", vote.ID().String()).Logger() + // drop stale votes + if vote.View < va.lowestRetainedView.Value() { + log.Debug().Msg("drop stale votes") + va.engineMetrics.InboundMessageDropped(metrics.EngineVoteAggregator, metrics.MessageBlockVote) + return + } + + // It's ok to silently drop votes in case our processing pipeline is full. + // It means that we are probably catching up. + if ok := va.queuedVotes.Push(vote); ok { + va.queuedMessagesNotifier.Notify() + } else { + log.Info().Msg("no queue capacity, dropping vote") + va.engineMetrics.InboundMessageDropped(metrics.EngineVoteAggregator, metrics.MessageBlockVote) + } +} + +// AddBlock notifies the VoteAggregator that it should start processing votes for the given block. +// The input block is queued internally within the `VoteAggregator` and processed _asynchronously_ +// by the VoteAggregator's internal worker routines. +// CAUTION: we expect that the input block's validity has been confirmed prior to calling AddBlock, +// including the proposer's signature. Otherwise, VoteAggregator might crash or exhibit undefined +// behaviour. +func (va *VoteAggregator) AddBlock(block *model.Proposal) { + // It's ok to silently drop blocks in case our processing pipeline is full. + // It means that we are probably catching up. + if ok := va.queuedBlocks.Push(block); ok { + va.queuedMessagesNotifier.Notify() + } else { + va.log.Debug().Msgf("dropping block %x because queue is full", block.Block.BlockID) + } +} + // InvalidBlock notifies the VoteAggregator about an invalid proposal, so that it -// can process votes for the invalid block and slash the voters. Expected error -// returns during normal operations: -// * mempool.DecreasingPruningHeightError if proposal's view has already been pruned +// can process votes for the invalid block and slash the voters. +// No errors are expected during normal operations func (va *VoteAggregator) InvalidBlock(proposal *model.Proposal) error { slashingVoteConsumer := func(vote *model.Vote) { if proposal.Block.BlockID == vote.BlockID { @@ -254,7 +317,7 @@ func (va *VoteAggregator) InvalidBlock(proposal *model.Proposal) error { collector, _, err := va.collectors.GetOrCreateCollector(block.View) if err != nil { // ignore if our routine is outdated and some other one has pruned collectors - if mempool.IsDecreasingPruningHeightError(err) { + if mempool.IsBelowPrunedThresholdError(err) { return nil } return fmt.Errorf("could not retrieve vote collector for view %d: %w", block.View, err) @@ -276,8 +339,7 @@ func (va *VoteAggregator) PruneUpToView(lowestRetainedView uint64) { } // OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// -// (1) Informs sealing.Core about finalization of respective block. +// It informs sealing.Core about finalization of respective block. // // CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages // from external nodes cannot be considered as inputs to this function diff --git a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go index bb670288357..792c42cbca5 100644 --- a/consensus/hotstuff/voteaggregator/vote_aggregator_test.go +++ b/consensus/hotstuff/voteaggregator/vote_aggregator_test.go @@ -9,9 +9,11 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/utils/unittest" ) @@ -29,52 +31,83 @@ type VoteAggregatorTestSuite struct { collectors *mocks.VoteCollectors consumer *mocks.Consumer stopAggregator context.CancelFunc + errs <-chan error } func (s *VoteAggregatorTestSuite) SetupTest() { var err error - s.collectors = &mocks.VoteCollectors{} - s.consumer = &mocks.Consumer{} - - ready := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - done := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() + s.collectors = mocks.NewVoteCollectors(s.T()) + s.consumer = mocks.NewConsumer(s.T()) s.collectors.On("Start", mock.Anything).Once() - s.collectors.On("Ready").Return(ready).Once() - s.collectors.On("Done").Return(done).Once() + unittest.ReadyDoneify(s.collectors) - s.aggregator, err = NewVoteAggregator(unittest.Logger(), s.consumer, 0, s.collectors) + metricsCollector := metrics.NewNoopCollector() + + s.aggregator, err = NewVoteAggregator( + unittest.Logger(), + metricsCollector, + metricsCollector, + metricsCollector, + s.consumer, + 0, + s.collectors, + ) require.NoError(s.T(), err) ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) + signalerCtx, errs := irrecoverable.WithSignaler(ctx) s.stopAggregator = cancel + s.errs = errs s.aggregator.Start(signalerCtx) unittest.RequireCloseBefore(s.T(), s.aggregator.Ready(), 100*time.Millisecond, "should close before timeout") } func (s *VoteAggregatorTestSuite) TearDownTest() { s.stopAggregator() - unittest.RequireCloseBefore(s.T(), s.aggregator.Done(), time.Second, "should close before timeout") + unittest.RequireCloseBefore(s.T(), s.aggregator.Done(), 10*time.Second, "should close before timeout") } // TestOnFinalizedBlock tests if finalized block gets processed when send through `VoteAggregator`. // Tests the whole processing pipeline. func (s *VoteAggregatorTestSuite) TestOnFinalizedBlock() { finalizedBlock := unittest.BlockHeaderFixture(unittest.HeaderWithView(100)) - s.collectors.On("PruneUpToView", finalizedBlock.View).Once() - s.aggregator.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock, finalizedBlock.View-1)) - require.Eventually(s.T(), - func() bool { - return s.collectors.AssertCalled(s.T(), "PruneUpToView", finalizedBlock.View) - }, time.Second, time.Millisecond*20) + done := make(chan struct{}) + s.collectors.On("PruneUpToView", finalizedBlock.View).Run(func(args mock.Arguments) { + close(done) + }).Once() + s.aggregator.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) + unittest.AssertClosesBefore(s.T(), done, time.Second) +} + +// TestProcessInvalidBlock tests that processing invalid block results in exception, when given as +// an input to AddBlock (only expects _valid_ blocks per API contract). +// The exception should be propagated to the VoteAggregator's internal `ComponentManager`. +func (s *VoteAggregatorTestSuite) TestProcessInvalidBlock() { + block := helper.MakeProposal( + helper.WithBlock( + helper.MakeBlock( + helper.WithBlockView(100), + ), + ), + ) + processed := make(chan struct{}) + collector := mocks.NewVoteCollector(s.T()) + collector.On("ProcessBlock", block).Run(func(_ mock.Arguments) { + close(processed) + }).Return(model.InvalidBlockError{}) + s.collectors.On("GetOrCreateCollector", block.Block.View).Return(collector, true, nil).Once() + + // submit block for processing + s.aggregator.AddBlock(block) + unittest.RequireCloseBefore(s.T(), processed, 100*time.Millisecond, "should close before timeout") + + // expect a thrown error + select { + case err := <-s.errs: + require.Error(s.T(), err) + require.False(s.T(), model.IsInvalidBlockError(err)) + case <-time.After(100 * time.Millisecond): + s.T().Fatalf("expected error but haven't received anything") + } } diff --git a/consensus/hotstuff/voteaggregator/vote_collectors.go b/consensus/hotstuff/voteaggregator/vote_collectors.go index 8a44d6c48ca..90a82cc91c5 100644 --- a/consensus/hotstuff/voteaggregator/vote_collectors.go +++ b/consensus/hotstuff/voteaggregator/vote_collectors.go @@ -52,12 +52,12 @@ func NewVoteCollectors(logger zerolog.Logger, lowestRetainedView uint64, workerP // GetOrCreateCollector retrieves the hotstuff.VoteCollector for the specified // view or creates one if none exists. -// - (collector, true, nil) if no collector can be found by the block ID, and a new collector was created. -// - (collector, false, nil) if the collector can be found by the block ID +// - (collector, true, nil) if no collector can be found by the view, and a new collector was created. +// - (collector, false, nil) if the collector can be found by the view // - (nil, false, error) if running into any exception creating the vote collector state machine // // Expected error returns during normal operations: -// - mempool.DecreasingPruningHeightError - in case view is lower than lowestRetainedView +// - mempool.BelowPrunedThresholdError - in case view is lower than lowestRetainedView func (v *VoteCollectors) GetOrCreateCollector(view uint64) (hotstuff.VoteCollector, bool, error) { cachedCollector, hasCachedCollector, err := v.getCollector(view) if err != nil { @@ -91,12 +91,12 @@ func (v *VoteCollectors) GetOrCreateCollector(view uint64) (hotstuff.VoteCollect // getCollector retrieves hotstuff.VoteCollector from local cache in concurrent safe way. // Performs check for lowestRetainedView. // Expected error returns during normal operations: -// - mempool.DecreasingPruningHeightError - in case view is lower than lowestRetainedView +// - mempool.BelowPrunedThresholdError - in case view is lower than lowestRetainedView func (v *VoteCollectors) getCollector(view uint64) (hotstuff.VoteCollector, bool, error) { v.lock.RLock() defer v.lock.RUnlock() if view < v.lowestRetainedView { - return nil, false, mempool.NewDecreasingPruningHeightErrorf("cannot retrieve collector for pruned view %d (lowest retained view %d)", view, v.lowestRetainedView) + return nil, false, mempool.NewBelowPrunedThresholdErrorf("cannot retrieve collector for pruned view %d (lowest retained view %d)", view, v.lowestRetainedView) } clr, found := v.collectors[view] diff --git a/consensus/hotstuff/voteaggregator/vote_collectors_test.go b/consensus/hotstuff/voteaggregator/vote_collectors_test.go index d0434a42931..ee721ea1521 100644 --- a/consensus/hotstuff/voteaggregator/vote_collectors_test.go +++ b/consensus/hotstuff/voteaggregator/vote_collectors_test.go @@ -62,13 +62,13 @@ func (s *VoteCollectorsTestSuite) prepareMockedCollector(view uint64) *mocks.Vot } // TestGetOrCreatorCollector_ViewLowerThanLowest tests a scenario where caller tries to create a collector with view -// lower than already pruned one. This should result in sentinel error `DecreasingPruningHeightError` +// lower than already pruned one. This should result in sentinel error `BelowPrunedThresholdError` func (s *VoteCollectorsTestSuite) TestGetOrCreatorCollector_ViewLowerThanLowest() { collector, created, err := s.collectors.GetOrCreateCollector(s.lowestLevel - 10) require.Nil(s.T(), collector) require.False(s.T(), created) require.Error(s.T(), err) - require.True(s.T(), mempool.IsDecreasingPruningHeightError(err)) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) } // TestGetOrCreateCollector_ValidCollector tests a happy path scenario where we try first to create and then retrieve cached collector. @@ -148,7 +148,7 @@ func (s *VoteCollectorsTestSuite) TestPruneUpToView() { for _, prunedView := range prunedViews { _, _, err := s.collectors.GetOrCreateCollector(prunedView) require.Error(s.T(), err) - require.True(s.T(), mempool.IsDecreasingPruningHeightError(err)) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) } for _, collector := range expectedCollectors { diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2.go index 101a5a1a1ad..69d6fb350af 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2.go @@ -28,7 +28,7 @@ import ( // by `votecollector.VoteProcessorFactory` which adds the logic to verify // the proposer's vote (decorator pattern). type combinedVoteProcessorFactoryBaseV2 struct { - committee hotstuff.Committee + committee hotstuff.DynamicCommittee onQCCreated hotstuff.OnQCCreated packer hotstuff.Packer } @@ -36,7 +36,7 @@ type combinedVoteProcessorFactoryBaseV2 struct { // Create creates CombinedVoteProcessorV2 for processing votes for the given block. // Caller must treat all errors as exceptions func (f *combinedVoteProcessorFactoryBaseV2) Create(log zerolog.Logger, block *model.Block) (hotstuff.VerifyingVoteProcessor, error) { - allParticipants, err := f.committee.Identities(block.BlockID) + allParticipants, err := f.committee.IdentitiesByBlock(block.BlockID) if err != nil { return nil, fmt.Errorf("error retrieving consensus participants at block %v: %w", block.BlockID, err) } @@ -56,7 +56,7 @@ func (f *combinedVoteProcessorFactoryBaseV2) Create(log zerolog.Logger, block *m } publicKeyShares := make([]crypto.PublicKey, 0, len(allParticipants)) - dkg, err := f.committee.DKG(block.BlockID) + dkg, err := f.committee.DKG(block.View) if err != nil { return nil, fmt.Errorf("could not get DKG info at block %v: %w", block.BlockID, err) } @@ -75,7 +75,10 @@ func (f *combinedVoteProcessorFactoryBaseV2) Create(log zerolog.Logger, block *m } rbRector := signature.NewRandomBeaconReconstructor(dkg, randomBeaconInspector) - minRequiredWeight := hotstuff.ComputeWeightThresholdForBuildingQC(allParticipants.TotalWeight()) + minRequiredWeight, err := f.committee.QuorumThresholdForView(block.View) + if err != nil { + return nil, fmt.Errorf("could not get weight threshold for view %d: %w", block.View, err) + } return NewCombinedVoteProcessor( log, @@ -231,7 +234,9 @@ func (p *CombinedVoteProcessorV2) Process(vote *model.Vote) error { } // checking of conditions for building QC are satisfied - if p.stakingSigAggtor.TotalWeight() < p.minRequiredWeight { + totalWeight := p.stakingSigAggtor.TotalWeight() + p.log.Debug().Msgf("processed vote, total weight=(%d), required=(%d)", totalWeight, p.minRequiredWeight) + if totalWeight < p.minRequiredWeight { return nil } if !p.rbRector.EnoughShares() { @@ -255,7 +260,7 @@ func (p *CombinedVoteProcessorV2) Process(vote *model.Vote) error { p.log.Info(). Uint64("view", qc.View). Hex("signers", qc.SignerIndices). - Msg("new qc has been created") + Msg("new QC has been created") p.onQCCreated(qc) @@ -305,7 +310,7 @@ func buildQCWithPackerAndSigData( block *model.Block, blockSigData *hotstuff.BlockSignatureData, ) (*flow.QuorumCertificate, error) { - signerIndices, sigData, err := packer.Pack(block.BlockID, blockSigData) + signerIndices, sigData, err := packer.Pack(block.View, blockSigData) if err != nil { return nil, fmt.Errorf("could not pack the block sig data: %w", err) diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go index a1fe965bb02..ef1fa25df85 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go @@ -15,6 +15,7 @@ import ( bootstrapDKG "github.com/onflow/flow-go/cmd/bootstrap/dkg" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/helper" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" @@ -400,7 +401,7 @@ func (s *CombinedVoteProcessorV2TestSuite) TestProcess_ConcurrentCreatingQC() { s.reconstructor.On("EnoughShares").Return(true) // at this point sending any vote should result in creating QC. - s.packer.On("Pack", s.proposal.Block.BlockID, mock.Anything).Return(unittest.RandomBytes(100), unittest.RandomBytes(128), nil) + s.packer.On("Pack", s.proposal.Block.View, mock.Anything).Return(unittest.RandomBytes(100), unittest.RandomBytes(128), nil) s.onQCCreatedState.On("onQCCreated", mock.Anything).Return(nil).Once() var startupWg, shutdownWg sync.WaitGroup @@ -493,7 +494,7 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCCorrectness(testifyT *testing mergedSignerIDs := make(flow.IdentifierList, 0) packedSigData := unittest.RandomBytes(128) pcker := &mockhotstuff.Packer{} - pcker.On("Pack", block.BlockID, mock.Anything).Run(func(args mock.Arguments) { + pcker.On("Pack", block.View, mock.Anything).Run(func(args mock.Arguments) { blockSigData := args.Get(1).(*hotstuff.BlockSignatureData) // check that aggregated signers are part of all votes signers @@ -520,12 +521,12 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCCorrectness(testifyT *testing // fill merged signers with collected signers mergedSignerIDs = append(expectedBlockSigData.StakingSigners, expectedBlockSigData.RandomBeaconSigners...) }).Return( - func(flow.Identifier, *hotstuff.BlockSignatureData) []byte { + func(uint64, *hotstuff.BlockSignatureData) []byte { signerIndices, _ := msig.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs) return signerIndices }, - func(flow.Identifier, *hotstuff.BlockSignatureData) []byte { return packedSigData }, - func(flow.Identifier, *hotstuff.BlockSignatureData) error { return nil }).Once() + func(uint64, *hotstuff.BlockSignatureData) []byte { return packedSigData }, + func(uint64, *hotstuff.BlockSignatureData) error { return nil }).Once() // track if QC was created qcCreated := atomic.NewBool(false) @@ -690,7 +691,7 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCLiveness(testifyT *testing.T) signerIndices, err := msig.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs) require.NoError(t, err) - pcker.On("Pack", block.BlockID, mock.Anything).Return(signerIndices, packedSigData, nil) + pcker.On("Pack", block.View, mock.Anything).Return(signerIndices, packedSigData, nil) // track if QC was created qcCreated := atomic.NewBool(false) @@ -860,9 +861,11 @@ func TestCombinedVoteProcessorV2_BuildVerifyQC(t *testing.T) { }) require.NoError(t, err) - committee := &mockhotstuff.Committee{} - committee.On("Identities", block.BlockID, mock.Anything).Return(allIdentities, nil) - committee.On("DKG", block.BlockID).Return(inmemDKG, nil) + committee := &mockhotstuff.DynamicCommittee{} + committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(allIdentities.TotalWeight()), nil) + committee.On("IdentitiesByEpoch", block.View).Return(allIdentities, nil) + committee.On("IdentitiesByBlock", block.BlockID).Return(allIdentities, nil) + committee.On("DKG", block.View).Return(inmemDKG, nil) votes := make([]*model.Vote, 0, len(allIdentities)) @@ -884,11 +887,10 @@ func TestCombinedVoteProcessorV2_BuildVerifyQC(t *testing.T) { // create verifier that will do crypto checks of created QC verifier := verification.NewCombinedVerifier(committee, packer) - forks := &mockhotstuff.Forks{} // create validator which will do compliance and crypto checked of created QC - validator := hotstuffvalidator.New(committee, forks, verifier) + validator := hotstuffvalidator.New(committee, verifier) // check if QC is valid against parent - err := validator.ValidateQC(qc, block) + err := validator.ValidateQC(qc) require.NoError(t, err) qcCreated = true @@ -935,11 +937,12 @@ func TestReadRandomSourceFromPackedQCV2(t *testing.T) { // making a mock block header := unittest.BlockHeaderFixture() - block := model.BlockFromFlow(header, header.View-1) + block := model.BlockFromFlow(header) // create a packer - committee := &mockhotstuff.Committee{} - committee.On("Identities", block.BlockID, mock.Anything).Return(allSigners, nil) + committee := &mockhotstuff.DynamicCommittee{} + committee.On("IdentitiesByBlock", block.BlockID).Return(allSigners, nil) + committee.On("IdentitiesByEpoch", block.View).Return(allSigners, nil) packer := signature.NewConsensusSigDataPacker(committee) qc, err := buildQCWithPackerAndSigData(packer, block, blockSigData) diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3.go index 7b03413cedd..1a2bdf72fee 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3.go @@ -30,7 +30,7 @@ import ( // the proposer's vote (decorator pattern). // nolint:unused type combinedVoteProcessorFactoryBaseV3 struct { - committee hotstuff.Committee + committee hotstuff.DynamicCommittee onQCCreated hotstuff.OnQCCreated packer hotstuff.Packer } @@ -39,7 +39,7 @@ type combinedVoteProcessorFactoryBaseV3 struct { // Caller must treat all errors as exceptions // nolint:unused func (f *combinedVoteProcessorFactoryBaseV3) Create(log zerolog.Logger, block *model.Block) (hotstuff.VerifyingVoteProcessor, error) { - allParticipants, err := f.committee.Identities(block.BlockID) + allParticipants, err := f.committee.IdentitiesByBlock(block.BlockID) if err != nil { return nil, fmt.Errorf("error retrieving consensus participants at block %v: %w", block.BlockID, err) } @@ -58,7 +58,7 @@ func (f *combinedVoteProcessorFactoryBaseV3) Create(log zerolog.Logger, block *m return nil, fmt.Errorf("could not create aggregator for staking signatures: %w", err) } - dkg, err := f.committee.DKG(block.BlockID) + dkg, err := f.committee.DKG(block.View) if err != nil { return nil, fmt.Errorf("could not get DKG info at block %v: %w", block.BlockID, err) } @@ -85,7 +85,10 @@ func (f *combinedVoteProcessorFactoryBaseV3) Create(log zerolog.Logger, block *m } rbRector := signature.NewRandomBeaconReconstructor(dkg, randomBeaconInspector) - minRequiredWeight := hotstuff.ComputeWeightThresholdForBuildingQC(allParticipants.TotalWeight()) + minRequiredWeight, err := f.committee.QuorumThresholdForView(block.View) + if err != nil { + return nil, fmt.Errorf("could not get weight threshold for view %d: %w", block.View, err) + } return &CombinedVoteProcessorV3{ log: log.With().Hex("block_id", block.BlockID[:]).Logger(), @@ -150,11 +153,11 @@ func (p *CombinedVoteProcessorV3) Status() hotstuff.VoteCollectorStatus { // `VerifyingVoteProcessor` (once as part of a cached vote, once as an individual vote). This can be exploited // by a byzantine proposer to be erroneously counted twice, which would lead to a safety fault. // -// TODO: (suggestion) I think it would be worth-while to include a second `votesCache` into the `CombinedVoteProcessorV3`. -// Thereby, `CombinedVoteProcessorV3` inherently guarantees correctness of the QCs it produces without relying on -// external conditions (making the code more modular, less interdependent and thereby easier to maintain). The -// runtime overhead is marginal: For `votesCache` to add 500 votes (concurrently with 20 threads) takes about -// 0.25ms. This runtime overhead is neglectable and a good tradeoff for the gain in maintainability and code clarity. +// TODO (suggestion): I think it would be worth-while to include a second `votesCache` into the `CombinedVoteProcessorV3`. +// Thereby, `CombinedVoteProcessorV3` inherently guarantees correctness of the QCs it produces without relying on +// external conditions (making the code more modular, less interdependent and thereby easier to maintain). The +// runtime overhead is marginal: For `votesCache` to add 500 votes (concurrently with 20 threads) takes about +// 0.25ms. This runtime overhead is neglectable and a good tradeoff for the gain in maintainability and code clarity. func (p *CombinedVoteProcessorV3) Process(vote *model.Vote) error { err := EnsureVoteForBlock(vote, p.block) if err != nil { @@ -255,7 +258,7 @@ func (p *CombinedVoteProcessorV3) Process(vote *model.Vote) error { p.log.Info(). Uint64("view", qc.View). Hex("signers", qc.SignerIndices). - Msg("new qc has been created") + Msg("new QC has been created") p.onQCCreated(qc) @@ -305,7 +308,7 @@ func (p *CombinedVoteProcessorV3) buildQC() (*flow.QuorumCertificate, error) { AggregatedRandomBeaconSig: aggregatedRandomBeaconSig, ReconstructedRandomBeaconSig: reconstructedBeaconSig, } - signerIndices, sigData, err := p.packer.Pack(p.block.BlockID, blockSigData) + signerIndices, sigData, err := p.packer.Pack(p.block.View, blockSigData) if err != nil { return nil, fmt.Errorf("could not pack the block sig data: %w", err) } diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go index 858f69f816e..01497d59ff5 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go @@ -15,6 +15,7 @@ import ( bootstrapDKG "github.com/onflow/flow-go/cmd/bootstrap/dkg" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/helper" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" @@ -399,7 +400,7 @@ func (s *CombinedVoteProcessorV3TestSuite) TestProcess_ConcurrentCreatingQC() { s.reconstructor.On("EnoughShares").Return(true) // at this point sending any vote should result in creating QC. - s.packer.On("Pack", s.proposal.Block.BlockID, mock.Anything).Return(unittest.RandomBytes(100), unittest.RandomBytes(128), nil) + s.packer.On("Pack", s.proposal.Block.View, mock.Anything).Return(unittest.RandomBytes(100), unittest.RandomBytes(128), nil) s.onQCCreatedState.On("onQCCreated", mock.Anything).Return(nil).Once() var startupWg, shutdownWg sync.WaitGroup @@ -514,7 +515,7 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing mergedSignerIDs := (flow.IdentifierList)(nil) packedSigData := unittest.RandomBytes(128) pcker := &mockhotstuff.Packer{} - pcker.On("Pack", block.BlockID, mock.Anything).Run(func(args mock.Arguments) { + pcker.On("Pack", block.View, mock.Anything).Run(func(args mock.Arguments) { blockSigData := args.Get(1).(*hotstuff.BlockSignatureData) // in the following, we check validity for each field of `blockSigData` individually @@ -566,12 +567,12 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing // fill merged signers with collected signers mergedSignerIDs = append(blockSigData.StakingSigners, blockSigData.RandomBeaconSigners...) }).Return( - func(flow.Identifier, *hotstuff.BlockSignatureData) []byte { + func(uint64, *hotstuff.BlockSignatureData) []byte { signerIndices, _ := signature.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs) return signerIndices }, - func(flow.Identifier, *hotstuff.BlockSignatureData) []byte { return packedSigData }, - func(flow.Identifier, *hotstuff.BlockSignatureData) error { return nil }).Once() + func(uint64, *hotstuff.BlockSignatureData) []byte { return packedSigData }, + func(uint64, *hotstuff.BlockSignatureData) error { return nil }).Once() // track if QC was created qcCreated := atomic.NewBool(false) @@ -727,7 +728,7 @@ func TestCombinedVoteProcessorV3_OnlyRandomBeaconSigners(testifyT *testing.T) { // Adding the vote should trigger QC generation. We expect `BlockSignatureData.StakingSigners` // and `BlockSignatureData.AggregatedStakingSig` to be both empty, as there are no staking signatures. - packer.On("Pack", block.BlockID, mock.Anything). + packer.On("Pack", block.View, mock.Anything). Run(func(args mock.Arguments) { blockSigData := args.Get(1).(*hotstuff.BlockSignatureData) require.Empty(testifyT, blockSigData.StakingSigners) @@ -824,7 +825,7 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness(testifyT *testing.T) signerIndices, err := signature.EncodeSignersToIndices(mergedSignerIDs, mergedSignerIDs) require.NoError(t, err) - pcker.On("Pack", block.BlockID, mock.Anything).Return(signerIndices, packedSigData, nil) + pcker.On("Pack", block.View, mock.Anything).Return(signerIndices, packedSigData, nil) // track if QC was created qcCreated := atomic.NewBool(false) @@ -995,9 +996,11 @@ func TestCombinedVoteProcessorV3_BuildVerifyQC(t *testing.T) { }) require.NoError(t, err) - committee := &mockhotstuff.Committee{} - committee.On("Identities", block.BlockID, mock.Anything).Return(allIdentities, nil) - committee.On("DKG", block.BlockID).Return(inmemDKG, nil) + committee := &mockhotstuff.DynamicCommittee{} + committee.On("IdentitiesByBlock", block.BlockID).Return(allIdentities, nil) + committee.On("IdentitiesByEpoch", block.View).Return(allIdentities, nil) + committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(allIdentities.TotalWeight()), nil) + committee.On("DKG", block.View).Return(inmemDKG, nil) votes := make([]*model.Vote, 0, len(allIdentities)) @@ -1019,11 +1022,10 @@ func TestCombinedVoteProcessorV3_BuildVerifyQC(t *testing.T) { // create verifier that will do crypto checks of created QC verifier := verification.NewCombinedVerifierV3(committee, packer) - forks := &mockhotstuff.Forks{} // create validator which will do compliance and crypto checked of created QC - validator := hotstuffvalidator.New(committee, forks, verifier) + validator := hotstuffvalidator.New(committee, verifier) // check if QC is valid against parent - err := validator.ValidateQC(qc, block) + err := validator.ValidateQC(qc) require.NoError(t, err) qcCreated = true diff --git a/consensus/hotstuff/votecollector/factory.go b/consensus/hotstuff/votecollector/factory.go index 91ce5b93f2e..31d36119978 100644 --- a/consensus/hotstuff/votecollector/factory.go +++ b/consensus/hotstuff/votecollector/factory.go @@ -63,7 +63,7 @@ func (f *VoteProcessorFactory) Create(log zerolog.Logger, proposal *model.Propos // NewStakingVoteProcessorFactory implements hotstuff.VoteProcessorFactory for // members of a collector cluster. For their cluster-local hotstuff, collectors // only sign with their staking key. -func NewStakingVoteProcessorFactory(committee hotstuff.Committee, onQCCreated hotstuff.OnQCCreated) *VoteProcessorFactory { +func NewStakingVoteProcessorFactory(committee hotstuff.DynamicCommittee, onQCCreated hotstuff.OnQCCreated) *VoteProcessorFactory { base := &stakingVoteProcessorFactoryBase{ committee: committee, onQCCreated: onQCCreated, @@ -74,7 +74,7 @@ func NewStakingVoteProcessorFactory(committee hotstuff.Committee, onQCCreated ho } // NewCombinedVoteProcessorFactory implements hotstuff.VoteProcessorFactory fo -// participants of the Main Consensus Committee. +// participants of the Main Consensus committee. // // With their vote, members of the main consensus committee can contribute to hotstuff and // the random beacon. When a consensus participant signs with its random beacon key, it @@ -82,7 +82,7 @@ func NewStakingVoteProcessorFactory(committee hotstuff.Committee, onQCCreated ho // participant can sign with its staking key; thereby it contributes only to consensus but // not the random beacon. There should be an economic incentive for the nodes to preferably // sign with their random beacon key. -func NewCombinedVoteProcessorFactory(committee hotstuff.Committee, onQCCreated hotstuff.OnQCCreated) *VoteProcessorFactory { +func NewCombinedVoteProcessorFactory(committee hotstuff.DynamicCommittee, onQCCreated hotstuff.OnQCCreated) *VoteProcessorFactory { base := &combinedVoteProcessorFactoryBaseV2{ committee: committee, onQCCreated: onQCCreated, @@ -99,7 +99,7 @@ func NewCombinedVoteProcessorFactory(committee hotstuff.Committee, onQCCreated h // suitable for the collector's local cluster consensus. // Intended use: only for bootstrapping. // UNSAFE: the proposer vote for `block` is _not_ validated or included -func NewBootstrapCombinedVoteProcessor(log zerolog.Logger, committee hotstuff.Committee, block *model.Block, onQCCreated hotstuff.OnQCCreated) (hotstuff.VerifyingVoteProcessor, error) { +func NewBootstrapCombinedVoteProcessor(log zerolog.Logger, committee hotstuff.DynamicCommittee, block *model.Block, onQCCreated hotstuff.OnQCCreated) (hotstuff.VerifyingVoteProcessor, error) { factory := &combinedVoteProcessorFactoryBaseV2{ committee: committee, onQCCreated: onQCCreated, @@ -112,7 +112,7 @@ func NewBootstrapCombinedVoteProcessor(log zerolog.Logger, committee hotstuff.Co // suitable for the collector's local cluster consensus. // Intended use: only for bootstrapping. // UNSAFE: the proposer vote for `block` is _not_ validated or included -func NewBootstrapStakingVoteProcessor(log zerolog.Logger, committee hotstuff.Committee, block *model.Block, onQCCreated hotstuff.OnQCCreated) (hotstuff.VerifyingVoteProcessor, error) { +func NewBootstrapStakingVoteProcessor(log zerolog.Logger, committee hotstuff.DynamicCommittee, block *model.Block, onQCCreated hotstuff.OnQCCreated) (hotstuff.VerifyingVoteProcessor, error) { factory := &stakingVoteProcessorFactoryBase{ committee: committee, onQCCreated: onQCCreated, diff --git a/consensus/hotstuff/votecollector/staking_vote_processor.go b/consensus/hotstuff/votecollector/staking_vote_processor.go index 63f33089cc0..a470d97bc67 100644 --- a/consensus/hotstuff/votecollector/staking_vote_processor.go +++ b/consensus/hotstuff/votecollector/staking_vote_processor.go @@ -28,14 +28,14 @@ import ( // by `votecollector.VoteProcessorFactory` which adds the logic to verify // the proposer's vote (decorator pattern). type stakingVoteProcessorFactoryBase struct { - committee hotstuff.Committee + committee hotstuff.DynamicCommittee onQCCreated hotstuff.OnQCCreated } // Create creates StakingVoteProcessor for processing votes for the given block. // Caller must treat all errors as exceptions func (f *stakingVoteProcessorFactoryBase) Create(log zerolog.Logger, block *model.Block) (hotstuff.VerifyingVoteProcessor, error) { - allParticipants, err := f.committee.Identities(block.BlockID) + allParticipants, err := f.committee.IdentitiesByBlock(block.BlockID) if err != nil { return nil, fmt.Errorf("error retrieving consensus participants: %w", err) } @@ -54,10 +54,13 @@ func (f *stakingVoteProcessorFactoryBase) Create(log zerolog.Logger, block *mode return nil, fmt.Errorf("could not create aggregator for staking signatures: %w", err) } - minRequiredWeight := hotstuff.ComputeWeightThresholdForBuildingQC(allParticipants.TotalWeight()) + minRequiredWeight, err := f.committee.QuorumThresholdForView(block.View) + if err != nil { + return nil, fmt.Errorf("could not get weight threshold for view %d: %w", block.View, err) + } return &StakingVoteProcessor{ - log: log, + log: log.With().Hex("block_id", block.BlockID[:]).Logger(), block: block, stakingSigAggtor: stakingSigAggtor, onQCCreated: f.onQCCreated, @@ -136,6 +139,8 @@ func (p *StakingVoteProcessor) Process(vote *model.Vote) error { return fmt.Errorf("unexpected exception adding signature from vote %x to staking aggregator: %w", vote.ID(), err) } + p.log.Debug().Msgf("processed vote, total weight=(%d), required=(%d)", totalWeight, p.minRequiredWeight) + // checking of conditions for building QC are satisfied if totalWeight < p.minRequiredWeight { return nil @@ -150,6 +155,11 @@ func (p *StakingVoteProcessor) Process(vote *model.Vote) error { if err != nil { return fmt.Errorf("internal error constructing QC from votes: %w", err) } + + p.log.Info(). + Uint64("view", qc.View). + Hex("signers", qc.SignerIndices). + Msg("new QC has been created") p.onQCCreated(qc) return nil diff --git a/consensus/hotstuff/votecollector/staking_vote_processor_test.go b/consensus/hotstuff/votecollector/staking_vote_processor_test.go index 7eeac8ae68b..b6efe8f93c4 100644 --- a/consensus/hotstuff/votecollector/staking_vote_processor_test.go +++ b/consensus/hotstuff/votecollector/staking_vote_processor_test.go @@ -11,6 +11,7 @@ import ( "go.uber.org/atomic" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/helper" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" @@ -270,8 +271,10 @@ func TestStakingVoteProcessorV2_BuildVerifyQC(t *testing.T) { block := helper.MakeBlock(helper.WithBlockView(view), helper.WithBlockProposer(leader.NodeID)) - committee := &mockhotstuff.Committee{} - committee.On("Identities", block.BlockID, mock.Anything).Return(stakingSigners, nil) + committee := &mockhotstuff.DynamicCommittee{} + committee.On("IdentitiesByEpoch", block.View).Return(stakingSigners, nil) + committee.On("IdentitiesByBlock", block.BlockID).Return(stakingSigners, nil) + committee.On("QuorumThresholdForView", mock.Anything).Return(committees.WeightThresholdToBuildQC(stakingSigners.TotalWeight()), nil) votes := make([]*model.Vote, 0, len(stakingSigners)) @@ -291,11 +294,10 @@ func TestStakingVoteProcessorV2_BuildVerifyQC(t *testing.T) { onQCCreated := func(qc *flow.QuorumCertificate) { // create verifier that will do crypto checks of created QC verifier := verification.NewStakingVerifier() - forks := &mockhotstuff.Forks{} // create validator which will do compliance and crypto checked of created QC - validator := hotstuffvalidator.New(committee, forks, verifier) + validator := hotstuffvalidator.New(committee, verifier) // check if QC is valid against parent - err := validator.ValidateQC(qc, block) + err := validator.ValidateQC(qc) require.NoError(t, err) qcCreated = true diff --git a/consensus/hotstuff/votecollector/statemachine.go b/consensus/hotstuff/votecollector/statemachine.go index cb1d5c85a80..6b7173196ab 100644 --- a/consensus/hotstuff/votecollector/statemachine.go +++ b/consensus/hotstuff/votecollector/statemachine.go @@ -63,7 +63,7 @@ func NewStateMachine( verifyingVoteProcessorFactory VerifyingVoteProcessorFactory, ) *VoteCollector { log = log.With(). - Str("hotstuff", "VoteCollector"). + Str("component", "hotstuff.vote_collector"). Uint64("view", view). Logger() sm := &VoteCollector{ @@ -130,14 +130,15 @@ func (m *VoteCollector) processVote(vote *model.Vote) error { currentState := processor.Status() err := processor.Process(vote) if err != nil { - if model.IsInvalidVoteError(err) { - m.notifier.OnInvalidVoteDetected(vote) + if invalidVoteErr, ok := model.AsInvalidVoteError(err); ok { + m.notifier.OnInvalidVoteDetected(*invalidVoteErr) return nil } // ATTENTION: due to how our logic is designed this situation is only possible // where we receive the same vote twice, this is not a case of double voting. // This scenario is possible if leader submits his vote additionally to the vote in proposal. if model.IsDuplicatedSignerError(err) { + m.log.Debug().Msgf("duplicated signer %x", vote.SignerID) return nil } return err @@ -147,6 +148,7 @@ func (m *VoteCollector) processVote(vote *model.Vote) error { continue } + m.notifier.OnVoteProcessed(vote) return nil } } @@ -274,7 +276,9 @@ func (m *VoteCollector) terminateVoteProcessing() { // processCachedVotes feeds all cached votes into the VoteProcessor func (m *VoteCollector) processCachedVotes(block *model.Block) { - for _, vote := range m.votesCache.All() { + cachedVotes := m.votesCache.All() + m.log.Info().Msgf("processing %d cached votes", len(cachedVotes)) + for _, vote := range cachedVotes { if vote.BlockID != block.BlockID { continue } diff --git a/consensus/hotstuff/votecollector/statemachine_test.go b/consensus/hotstuff/votecollector/statemachine_test.go index bc59c9a55b5..8ad19e98903 100644 --- a/consensus/hotstuff/votecollector/statemachine_test.go +++ b/consensus/hotstuff/votecollector/statemachine_test.go @@ -8,6 +8,7 @@ import ( "github.com/gammazero/workerpool" "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -63,13 +64,13 @@ func (s *StateMachineTestSuite) SetupTest() { // prepareMockedProcessor prepares a mocked processor and stores it in map, later it will be used // to mock behavior of verifying vote processor. -func (s *StateMachineTestSuite) prepareMockedProcessor(block *model.Block) *mocks.VerifyingVoteProcessor { +func (s *StateMachineTestSuite) prepareMockedProcessor(proposal *model.Proposal) *mocks.VerifyingVoteProcessor { processor := &mocks.VerifyingVoteProcessor{} processor.On("Block").Return(func() *model.Block { - return block + return proposal.Block }).Maybe() processor.On("Status").Return(hotstuff.VoteCollectorStatusVerifying) - s.mockedProcessors[block.BlockID] = processor + s.mockedProcessors[proposal.Block.BlockID] = processor return processor } @@ -78,7 +79,7 @@ func (s *StateMachineTestSuite) prepareMockedProcessor(block *model.Block) *mock func (s *StateMachineTestSuite) TestStatus_StateTransitions() { block := helper.MakeBlock(helper.WithBlockView(s.view)) proposal := helper.MakeProposal(helper.WithBlock(block)) - s.prepareMockedProcessor(block) + s.prepareMockedProcessor(proposal) // by default, we should create in caching status require.Equal(s.T(), hotstuff.VoteCollectorStatusCaching, s.collector.Status()) @@ -116,11 +117,12 @@ func (s *StateMachineTestSuite) Test_FactoryErrorPropagation() { func (s *StateMachineTestSuite) TestAddVote_VerifyingState() { block := helper.MakeBlock(helper.WithBlockView(s.view)) proposal := helper.MakeProposal(helper.WithBlock(block)) - processor := s.prepareMockedProcessor(block) + processor := s.prepareMockedProcessor(proposal) err := s.collector.ProcessBlock(proposal) require.NoError(s.T(), err) s.T().Run("add-valid-vote", func(t *testing.T) { vote := unittest.VoteForBlockFixture(block) + s.notifier.On("OnVoteProcessed", vote).Once() processor.On("Process", vote).Return(nil).Once() err := s.collector.AddVote(vote) require.NoError(t, err) @@ -128,6 +130,7 @@ func (s *StateMachineTestSuite) TestAddVote_VerifyingState() { }) s.T().Run("add-double-vote", func(t *testing.T) { firstVote := unittest.VoteForBlockFixture(block) + s.notifier.On("OnVoteProcessed", firstVote).Once() processor.On("Process", firstVote).Return(nil).Once() err := s.collector.AddVote(firstVote) require.NoError(t, err) @@ -149,18 +152,22 @@ func (s *StateMachineTestSuite) TestAddVote_VerifyingState() { s.T().Run("add-invalid-vote", func(t *testing.T) { vote := unittest.VoteForBlockFixture(block, unittest.WithVoteView(s.view)) processor.On("Process", vote).Return(model.NewInvalidVoteErrorf(vote, "")).Once() - - s.notifier.On("OnInvalidVoteDetected", vote).Return(nil).Once() + s.notifier.On("OnVoteProcessed", vote).Once() + s.notifier.On("OnInvalidVoteDetected", mock.Anything).Run(func(args mock.Arguments) { + invalidVoteErr := args.Get(0).(model.InvalidVoteError) + require.Equal(s.T(), vote, invalidVoteErr.Vote) + }).Return(nil).Once() err := s.collector.AddVote(vote) // in case process returns model.InvalidVoteError we should silently ignore this error require.NoError(t, err) // but should get notified about invalid vote - s.notifier.AssertCalled(t, "OnInvalidVoteDetected", vote) + s.notifier.AssertCalled(t, "OnInvalidVoteDetected", mock.Anything) processor.AssertCalled(t, "Process", vote) }) s.T().Run("add-repeated-vote", func(t *testing.T) { vote := unittest.VoteForBlockFixture(block) + s.notifier.On("OnVoteProcessed", vote).Once() processor.On("Process", vote).Return(nil).Once() err := s.collector.AddVote(vote) require.NoError(t, err) @@ -199,10 +206,12 @@ func (s *StateMachineTestSuite) TestProcessBlock_ProcessingOfCachedVotes() { votes := 10 block := helper.MakeBlock(helper.WithBlockView(s.view)) proposal := helper.MakeProposal(helper.WithBlock(block)) - processor := s.prepareMockedProcessor(block) + processor := s.prepareMockedProcessor(proposal) for i := 0; i < votes; i++ { vote := unittest.VoteForBlockFixture(block) - // eventually it has to be process by processor + // once when caching vote, and once when processing cached vote + s.notifier.On("OnVoteProcessed", vote).Twice() + // eventually it has to be processed by processor processor.On("Process", vote).Return(nil).Once() require.NoError(s.T(), s.collector.AddVote(vote)) } @@ -219,7 +228,8 @@ func (s *StateMachineTestSuite) TestProcessBlock_ProcessingOfCachedVotes() { // are propagated up the call stack (potentially wrapped), but are not replaced. func (s *StateMachineTestSuite) Test_VoteProcessorErrorPropagation() { block := helper.MakeBlock(helper.WithBlockView(s.view)) - processor := s.prepareMockedProcessor(block) + proposal := helper.MakeProposal(helper.WithBlock(block)) + processor := s.prepareMockedProcessor(proposal) err := s.collector.ProcessBlock(helper.MakeProposal(helper.WithBlock(block))) require.NoError(s.T(), err) @@ -236,7 +246,8 @@ func (s *StateMachineTestSuite) Test_VoteProcessorErrorPropagation() { func (s *StateMachineTestSuite) RegisterVoteConsumer() { votes := 10 block := helper.MakeBlock(helper.WithBlockView(s.view)) - processor := s.prepareMockedProcessor(block) + proposal := helper.MakeProposal(helper.WithBlock(block)) + processor := s.prepareMockedProcessor(proposal) expectedVotes := make([]*model.Vote, 0) for i := 0; i < votes; i++ { vote := unittest.VoteForBlockFixture(block) diff --git a/consensus/hotstuff/voter.go b/consensus/hotstuff/voter.go deleted file mode 100644 index ef0b772c1f1..00000000000 --- a/consensus/hotstuff/voter.go +++ /dev/null @@ -1,17 +0,0 @@ -package hotstuff - -import ( - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// Voter produces votes for the given block according to voting rules. -type Voter interface { - // ProduceVoteIfVotable takes a block and current view, and decides whether to vote for the block. - // Returns: - // * (vote, nil): On the _first_ block for the current view that is safe to vote for. - // Subsequently, voter does _not_ vote for any other block with the same (or lower) view. - // * (nil, model.NoVoteError): If the voter decides that it does not want to vote for the given block. - // This is a sentinel error and _expected_ during normal operation. - // All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). - ProduceVoteIfVotable(block *model.Block, curView uint64) (*model.Vote, error) -} diff --git a/consensus/hotstuff/voter/voter.go b/consensus/hotstuff/voter/voter.go deleted file mode 100644 index b295d0fc7b7..00000000000 --- a/consensus/hotstuff/voter/voter.go +++ /dev/null @@ -1,86 +0,0 @@ -package voter - -import ( - "fmt" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/model" -) - -// Voter produces votes for the given block -type Voter struct { - signer hotstuff.Signer - forks hotstuff.ForksReader - persist hotstuff.Persister - committee hotstuff.Committee // only produce votes when we are valid committee members - lastVotedView uint64 // need to keep track of the last view we voted for so we don't double vote accidentally -} - -// New creates a new Voter instance -func New( - signer hotstuff.Signer, - forks hotstuff.ForksReader, - persist hotstuff.Persister, - committee hotstuff.Committee, - lastVotedView uint64, -) *Voter { - - return &Voter{ - signer: signer, - forks: forks, - persist: persist, - committee: committee, - lastVotedView: lastVotedView, - } -} - -// ProduceVoteIfVotable will make a decision on whether it will vote for the given proposal, the returned -// error indicates whether to vote or not. -// In order to ensure that only a safe node will be voted, Voter will ask Forks whether a vote is a safe node or not. -// The curView is taken as input to ensure Voter will only vote for proposals at current view and prevent double voting. -// Returns: -// - (vote, nil): On the _first_ block for the current view that is safe to vote for. -// Subsequently, voter does _not_ vote for any other block with the same (or lower) view. -// - (nil, model.NoVoteError): If the voter decides that it does not want to vote for the given block. -// This is a sentinel error and _expected_ during normal operation. -// -// All other errors are unexpected and potential symptoms of uncovered edge cases or corrupted internal state (fatal). -func (v *Voter) ProduceVoteIfVotable(block *model.Block, curView uint64) (*model.Vote, error) { - // sanity checks: - if curView != block.View { - return nil, fmt.Errorf("expecting block for current view %d, but block's view is %d", curView, block.View) - } - if curView <= v.lastVotedView { - return nil, fmt.Errorf("current view (%d) must be larger than the last voted view (%d)", curView, v.lastVotedView) - } - - // Do not produce a vote for blocks where we are not a valid committee member. - // HotStuff will ask for a vote for the first block of the next epoch, even if we - // have zero weight in the next epoch. Such vote can't be used to produce valid QCs. - _, err := v.committee.Identity(block.BlockID, v.committee.Self()) - if model.IsInvalidSignerError(err) { - return nil, model.NoVoteError{Msg: "not voting committee member for block"} - } - if err != nil { - return nil, fmt.Errorf("could not get self identity: %w", err) - } - - // generate vote if block is safe to vote for - if !v.forks.IsSafeBlock(block) { - return nil, model.NoVoteError{Msg: "not safe block"} - } - vote, err := v.signer.CreateVote(block) - if err != nil { - return nil, fmt.Errorf("could not vote for block: %w", err) - } - - // vote for the current view has been produced, update lastVotedView - // to prevent from voting for the same view again - v.lastVotedView = curView - err = v.persist.PutVoted(curView) - if err != nil { - return nil, fmt.Errorf("could not persist last voted: %w", err) - } - - return vote, nil -} diff --git a/consensus/hotstuff/voter/voter_test.go b/consensus/hotstuff/voter/voter_test.go deleted file mode 100644 index 80eece889f2..00000000000 --- a/consensus/hotstuff/voter/voter_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package voter - -import ( - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/consensus/hotstuff/helper" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/utils/unittest" -) - -func TestProduceVote(t *testing.T) { - t.Run("should vote for block", testVoterOK) - t.Run("should not vote for unsafe block", testUnsafe) - t.Run("should not vote for block with its view below the current view", testBelowVote) - t.Run("should not vote for block with its view above the current view", testAboveVote) - t.Run("should not vote for block with the same view as the last voted view", testEqualLastVotedView) - t.Run("should not vote for block with its view below the last voted view", testBelowLastVotedView) - t.Run("should not vote for the same view again", testVotingAgain) - t.Run("should not vote while not a committee member", testVotingWhileNonCommitteeMember) -} - -func createVoter(blockView uint64, lastVotedView uint64, isBlockSafe, isCommitteeMember bool) (*model.Block, *model.Vote, *Voter) { - block := helper.MakeBlock(helper.WithBlockView(blockView)) - expectVote := makeVote(block) - - forks := &mocks.ForksReader{} - forks.On("IsSafeBlock", block).Return(isBlockSafe) - - persist := &mocks.Persister{} - persist.On("PutVoted", mock.Anything).Return(nil) - - signer := &mocks.Signer{} - signer.On("CreateVote", mock.Anything).Return(expectVote, nil) - - committee := &mocks.Committee{} - me := unittest.IdentityFixture() - committee.On("Self").Return(me.NodeID, nil) - if isCommitteeMember { - committee.On("Identity", mock.Anything, me.NodeID).Return(me, nil) - } else { - committee.On("Identity", mock.Anything, me.NodeID).Return(nil, model.NewInvalidSignerErrorf("")) - } - - voter := New(signer, forks, persist, committee, lastVotedView) - return block, expectVote, voter -} - -func testVoterOK(t *testing.T) { - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(2), true, true - - // create voter - block, expectVote, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - // produce vote - vote, err := voter.ProduceVoteIfVotable(block, curView) - - require.NoError(t, err) - require.Equal(t, vote, expectVote) -} - -func testUnsafe(t *testing.T) { - // create unsafe block - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(2), false, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - _, err := voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "not safe") - require.True(t, model.IsNoVoteError(err)) -} - -func testBelowVote(t *testing.T) { - // curView < blockView - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(2), uint64(2), true, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - _, err := voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "expecting block for current view") - require.False(t, model.IsNoVoteError(err)) -} - -func testAboveVote(t *testing.T) { - // curView > blockView - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(4), uint64(2), true, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - _, err := voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "expecting block for current view") - require.False(t, model.IsNoVoteError(err)) -} - -func testEqualLastVotedView(t *testing.T) { - // curView == lastVotedView - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(3), true, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - _, err := voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "must be larger than the last voted view") - require.False(t, model.IsNoVoteError(err)) -} - -func testBelowLastVotedView(t *testing.T) { - // curView < lastVotedView - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(4), true, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - _, err := voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "must be larger than the last voted view") - require.False(t, model.IsNoVoteError(err)) -} - -func testVotingAgain(t *testing.T) { - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(2), true, true - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - // produce vote - _, err := voter.ProduceVoteIfVotable(block, curView) - require.NoError(t, err) - - // produce vote again for the same view - _, err = voter.ProduceVoteIfVotable(block, curView) - require.Error(t, err) - require.Contains(t, err.Error(), "must be larger than the last voted view") - require.False(t, model.IsNoVoteError(err)) -} - -func testVotingWhileNonCommitteeMember(t *testing.T) { - blockView, curView, lastVotedView, isBlockSafe, isCommitteeMember := uint64(3), uint64(3), uint64(2), true, false - - // create voter - block, _, voter := createVoter(blockView, lastVotedView, isBlockSafe, isCommitteeMember) - - // produce vote - _, err := voter.ProduceVoteIfVotable(block, curView) - - require.Error(t, err) - require.True(t, model.IsNoVoteError(err)) -} - -func makeVote(block *model.Block) *model.Vote { - return &model.Vote{ - BlockID: block.BlockID, - View: block.View, - SigData: nil, // signature doesn't matter in this test case - } -} diff --git a/consensus/integration/blockordelay_test.go b/consensus/integration/blockordelay_test.go index 4ec7ad15ff7..e3fde5817f5 100644 --- a/consensus/integration/blockordelay_test.go +++ b/consensus/integration/blockordelay_test.go @@ -1,78 +1,88 @@ -//go:build timesensitivetest -// +build timesensitivetest - package integration_test import ( + "fmt" "math/rand" + "sync" "time" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/network/channels" ) // This file includes functions to simulate network conditions. // The network conditions are simulated by defining whether a message sent to a receiver should be // blocked or delayed. -// block all messages sent by or received by a list of denied nodes for the first N messages -func blockNodesForFirstNMessages(n int, denyList ...*Node) BlockOrDelayFunc { - denyDict := make(map[flow.Identifier]*Node, len(denyList)) - for _, n := range denyList { - denyDict[n.id.ID()] = n +// blockNodesFirstMessages blocks the _first_ n incoming messages for each member in `denyList` +// (messages are counted individually for each Node). +func blockNodesFirstMessages(n uint64, denyList ...*Node) BlockOrDelayFunc { + blackList := make(map[flow.Identifier]uint64, len(denyList)) + for _, node := range denyList { + blackList[node.id.ID()] = n } - - sent, received := 0, 0 - - return func(channelID string, event interface{}, sender, receiver *Node) (bool, time.Duration) { - block, notBlock := true, false - - switch m := event.(type) { + lock := new(sync.Mutex) + return func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { + // filter only consensus messages + switch event.(type) { case *messages.BlockProposal: case *messages.BlockVote: case *messages.BlockResponse: - log := receiver.log.With().Int("blocks", len(m.Blocks)).Uint64("first", m.Blocks[0].Header.View). - Uint64("last", m.Blocks[len(m.Blocks)-1].Header.View).Logger() - log.Info().Msg("receives BlockResponse") - case *messages.SyncRequest: - case *messages.SyncResponse: - case *messages.RangeRequest: - case *messages.BatchRequest: + case *messages.TimeoutObject: default: - return notBlock, 0 - } - - if _, ok := denyDict[sender.id.ID()]; ok { - if sent >= n { - return notBlock, 0 - } - sent++ - return block, 0 + return false, 0 } - if _, ok := denyDict[receiver.id.ID()]; ok { - if received >= n { - return notBlock, 0 - } - received++ - return block, 0 + lock.Lock() + defer lock.Unlock() + count, ok := blackList[receiver.id.ID()] + if ok && count > 0 { + blackList[receiver.id.ID()] = count - 1 + return true, 0 } return false, 0 } } -func blockReceiverMessagesByPercentage(percent int) BlockOrDelayFunc { - rand.Seed(time.Now().UnixNano()) - return func(channelID string, event interface{}, sender, receiver *Node) (bool, time.Duration) { - block := rand.Intn(100) <= percent +// blockReceiverMessagesRandomly drops messages randomly with a probability of `dropProbability` ∈ [0,1] +func blockReceiverMessagesRandomly(dropProbability float32) BlockOrDelayFunc { + lock := new(sync.Mutex) + prng := rand.New(rand.NewSource(time.Now().UnixNano())) + return func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { + lock.Lock() + block := prng.Float32() < dropProbability + lock.Unlock() return block, 0 } } +// delayReceiverMessagesByRange delivers all messages, but with a randomly sampled +// delay in the interval [low, high). Panics when `low` < 0 or `low > `high` func delayReceiverMessagesByRange(low time.Duration, high time.Duration) BlockOrDelayFunc { - rand.Seed(time.Now().UnixNano()) - return func(channelID string, event interface{}, sender, receiver *Node) (bool, time.Duration) { - rng := high - low - delay := int64(low) + rand.Int63n(int64(rng)) - return false, time.Duration(delay) + lock := new(sync.Mutex) + prng := rand.New(rand.NewSource(time.Now().UnixNano())) + delayRangeNs := int64(high - low) + minDelayNs := int64(low) + + // fail early for non-sensical parameter settings + if int64(low) < 0 { + panic(fmt.Sprintf("minimal delay cannot be negative, but is %d ns", int64(low))) + } + if delayRangeNs < 0 { + panic(fmt.Sprintf("upper bound on delay (%d ns) cannot be smaller than lower bound (%d ns)", int64(high), int64(low))) + } + + // shortcut for low = high: always return low + if delayRangeNs == 0 { + return func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { + return false, low + } + } + // general version + return func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { + lock.Lock() + d := prng.Int63n(delayRangeNs) + lock.Unlock() + return false, time.Duration(minDelayNs + d) } } diff --git a/consensus/integration/epoch_test.go b/consensus/integration/epoch_test.go index 23d7639527b..aa41de368fe 100644 --- a/consensus/integration/epoch_test.go +++ b/consensus/integration/epoch_test.go @@ -1,7 +1,6 @@ package integration_test import ( - "context" "testing" "time" @@ -14,7 +13,6 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/flow/mapfunc" "github.com/onflow/flow-go/model/flow/order" - "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol/inmem" "github.com/onflow/flow-go/utils/unittest" ) @@ -32,7 +30,15 @@ func TestUnweightedNode(t *testing.T) { // add a consensus node to next epoch (it will have 0 weight in the current epoch) nextEpochParticipantsData := createConsensusIdentities(t, 1) - nextEpochIdentities := unittest.CompleteIdentitySet(nextEpochParticipantsData.Identities()...) + // epoch 2 identities includes: + // * same collection node from epoch 1, so cluster QCs are consistent + // * 1 new consensus node, joining at epoch 2 + // * random nodes with other roles + nextEpochIdentities := unittest.CompleteIdentitySet( + append( + rootSnapshot.Encodable().Identities.Filter(filter.HasRole(flow.RoleCollection)), + nextEpochParticipantsData.Identities()...)..., + ) rootSnapshot = withNextEpoch( rootSnapshot, nextEpochIdentities, @@ -41,21 +47,15 @@ func TestUnweightedNode(t *testing.T) { 10_000, ) - nodes, hub := createNodes(t, consensusParticipants, rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, consensusParticipants, rootSnapshot, stopper) hub.WithFilter(blockNothing) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - - runNodes(signalerCtx, nodes) - - unittest.AssertClosesBefore(t, stopper.stopped, 60*time.Second) + runFor(60 * time.Second) allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } @@ -81,16 +81,11 @@ func TestStaticEpochTransition(t *testing.T) { 4, ) - nodes, hub := createNodes(t, consensusParticipants, rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, consensusParticipants, rootSnapshot, stopper) hub.WithFilter(blockNothing) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - - runNodes(signalerCtx, nodes) - - unittest.AssertClosesBefore(t, stopper.stopped, 30*time.Second) + runFor(30 * time.Second) allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) @@ -101,7 +96,6 @@ func TestStaticEpochTransition(t *testing.T) { require.NoError(t, err) assert.Equal(t, firstEpochCounter+1, afterCounter) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } @@ -140,16 +134,11 @@ func TestEpochTransition_IdentitiesOverlap(t *testing.T) { 4, ) - nodes, hub := createNodes(t, consensusParticipants, rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, consensusParticipants, rootSnapshot, stopper) hub.WithFilter(blockNothing) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - - runNodes(signalerCtx, nodes) - - unittest.AssertClosesBefore(t, stopper.stopped, 30*time.Second) + runFor(30 * time.Second) allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) @@ -160,7 +149,6 @@ func TestEpochTransition_IdentitiesOverlap(t *testing.T) { require.NoError(t, err) assert.Equal(t, firstEpochCounter+1, afterCounter) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } @@ -195,16 +183,11 @@ func TestEpochTransition_IdentitiesDisjoint(t *testing.T) { 4, ) - nodes, hub := createNodes(t, consensusParticipants, rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, consensusParticipants, rootSnapshot, stopper) hub.WithFilter(blockNothing) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - - runNodes(signalerCtx, nodes) - - unittest.AssertClosesBefore(t, stopper.stopped, 60*time.Second) + runFor(60 * time.Second) allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) @@ -215,7 +198,6 @@ func TestEpochTransition_IdentitiesDisjoint(t *testing.T) { require.NoError(t, err) assert.Equal(t, firstEpochCounter+1, afterCounter) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } diff --git a/consensus/integration/integration_test.go b/consensus/integration/integration_test.go index 9db7c687cfc..6ba804d103d 100644 --- a/consensus/integration/integration_test.go +++ b/consensus/integration/integration_test.go @@ -18,8 +18,13 @@ import ( func runNodes(signalerCtx irrecoverable.SignalerContext, nodes []*Node) { for _, n := range nodes { go func(n *Node) { - n.aggregator.Start(signalerCtx) - <-util.AllReady(n.aggregator, n.compliance, n.sync) + n.committee.Start(signalerCtx) + n.hot.Start(signalerCtx) + n.voteAggregator.Start(signalerCtx) + n.timeoutAggregator.Start(signalerCtx) + n.compliance.Start(signalerCtx) + n.messageHub.Start(signalerCtx) + <-util.AllReady(n.committee, n.hot, n.voteAggregator, n.timeoutAggregator, n.compliance, n.sync, n.messageHub) }(n) } } @@ -28,9 +33,17 @@ func stopNodes(t *testing.T, cancel context.CancelFunc, nodes []*Node) { stoppingNodes := make([]<-chan struct{}, 0) cancel() for _, n := range nodes { - stoppingNodes = append(stoppingNodes, util.AllDone(n.aggregator, n.compliance, n.sync)) + stoppingNodes = append(stoppingNodes, util.AllDone( + n.committee, + n.hot, + n.voteAggregator, + n.timeoutAggregator, + n.compliance, + n.sync, + n.messageHub, + )) } - unittest.RequireCloseBefore(t, util.AllClosed(stoppingNodes...), time.Minute, "requiring nodes to stop") + unittest.RequireCloseBefore(t, util.AllClosed(stoppingNodes...), time.Second, "requiring nodes to stop") } // happy path: with 3 nodes, they can reach consensus @@ -38,39 +51,29 @@ func Test3Nodes(t *testing.T) { stopper := NewStopper(5, 0) participantsData := createConsensusIdentities(t, 3) rootSnapshot := createRootSnapshot(t, participantsData) - nodes, hub := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) hub.WithFilter(blockNothing) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - - runNodes(signalerCtx, nodes) - - unittest.AssertClosesBefore(t, stopper.stopped, 30*time.Second) + runFor(30 * time.Second) allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } // with 5 nodes, and one node completely blocked, the other 4 nodes can still reach consensus func Test5Nodes(t *testing.T) { - // 4 nodes should be able finalize at least 3 blocks. + // 4 nodes should be able to finalize at least 3 blocks. stopper := NewStopper(2, 1) participantsData := createConsensusIdentities(t, 5) rootSnapshot := createRootSnapshot(t, participantsData) - nodes, hub := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) hub.WithFilter(blockNodes(nodes[0])) - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - runNodes(signalerCtx, nodes) - - <-stopper.stopped + runFor(30 * time.Second) header, err := nodes[0].state.Final().Head() require.NoError(t, err) @@ -81,7 +84,6 @@ func Test5Nodes(t *testing.T) { allViews := allFinalizedViews(t, nodes[1:]) assertSafety(t, allViews) - stopNodes(t, cancel, nodes) cleanupNodes(nodes) } @@ -127,8 +129,7 @@ func chainViews(t *testing.T, node *Node) []uint64 { require.NoError(t, err) } - // reverse all views to start from lower view to higher view - + // reverse all views to runFor from lower view to higher view low2high := make([]uint64, 0) for i := len(views) - 1; i >= 0; i-- { low2high = append(low2high, views[i]) @@ -136,27 +137,35 @@ func chainViews(t *testing.T, node *Node) []uint64 { return low2high } +// BlockOrDelayFunc is a function for deciding whether a message (or other event) should be +// blocked or delayed. The first return value specifies whether the event should be dropped +// entirely (return value `true`) or should be delivered (return value `false`). The second +// return value specifies the delay by which the message should be delivered. +// Implementations must be CONCURRENCY SAFE. type BlockOrDelayFunc func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) -// block nothing -func blockNothing(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { +// blockNothing specifies that _all_ messages should be delivered without delay. +// I.e. this function returns always `false` (no blocking), `0` (no delay). +func blockNothing(_ channels.Channel, _ interface{}, _, _ *Node) (bool, time.Duration) { return false, 0 } -// block all messages sent by or received by a list of denied nodes +// blockNodes specifies that all messages sent or received by any member of the `denyList` +// should be dropped, i.e. we return `true` (block message), `0` (no delay). +// For nodes _not_ in the `denyList`, we return `false` (no blocking), `0` (no delay). func blockNodes(denyList ...*Node) BlockOrDelayFunc { - blackList := make(map[flow.Identifier]*Node, len(denyList)) + denyMap := make(map[flow.Identifier]*Node, len(denyList)) for _, n := range denyList { - blackList[n.id.ID()] = n + denyMap[n.id.ID()] = n } + // no concurrency protection needed as blackList is only read but not modified return func(channel channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { - block, notBlock := true, false - if _, ok := blackList[sender.id.ID()]; ok { - return block, 0 + if _, ok := denyMap[sender.id.ID()]; ok { + return true, 0 // block the message } - if _, ok := blackList[receiver.id.ID()]; ok { - return block, 0 + if _, ok := denyMap[receiver.id.ID()]; ok { + return true, 0 // block the message } - return notBlock, 0 + return false, 0 // allow the message } } diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index bea576fb42a..a275499119d 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -1,6 +1,7 @@ package integration_test import ( + "context" "fmt" "os" "sort" @@ -22,12 +23,15 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/persister" hsig "github.com/onflow/flow-go/consensus/hotstuff/signature" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutaggregator" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutcollector" "github.com/onflow/flow-go/consensus/hotstuff/verification" "github.com/onflow/flow-go/consensus/hotstuff/voteaggregator" "github.com/onflow/flow-go/consensus/hotstuff/votecollector" "github.com/onflow/flow-go/crypto" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/consensus/compliance" + "github.com/onflow/flow-go/engine/consensus/message_hub" "github.com/onflow/flow-go/model/bootstrap" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -38,13 +42,14 @@ import ( synccore "github.com/onflow/flow-go/module/chainsync" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" "github.com/onflow/flow-go/module/id" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/local" consensusMempools "github.com/onflow/flow-go/module/mempool/consensus" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" + msig "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/state/protocol" bprotocol "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/blocktimer" @@ -56,7 +61,7 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -const hotstuffTimeout = 100 * time.Millisecond +const hotstuffTimeout = 500 * time.Millisecond // RandomBeaconNodeInfo stores information about participation in DKG process for consensus node // contains private + public keys and participant index @@ -129,23 +134,21 @@ func (p *ConsensusParticipants) Update(epochCounter uint64, data *run.Participan } type Node struct { - db *badger.DB - dbDir string - index int - log zerolog.Logger - id *flow.Identity - compliance *compliance.Engine - sync *synceng.Engine - hot module.HotStuff - aggregator hotstuff.VoteAggregator - state *bprotocol.MutableState - headers *storage.Headers - net *Network -} - -func (n *Node) Shutdown() { - <-n.sync.Done() - <-n.compliance.Done() + db *badger.DB + dbDir string + index int + log zerolog.Logger + id *flow.Identity + compliance *compliance.Engine + sync *synceng.Engine + hot module.HotStuff + committee *committees.Consensus + voteAggregator hotstuff.VoteAggregator + timeoutAggregator hotstuff.TimeoutAggregator + messageHub *message_hub.MessageHub + state *bprotocol.MutableState + headers *storage.Headers + net *Network } // epochInfo is a helper structure for storing epoch information such as counter and final view @@ -177,17 +180,13 @@ func buildEpochLookupList(epochs ...protocol.Epoch) []epochInfo { return infos } -// Inputs: -// - n: the total number of nodes to be created -// - finalizedCount: the number of finalized blocks before stopping the tests -// - tolerate: the number of node to tolerate that don't need to reach the finalization count -// before stopping the tests -func createNodes( - t *testing.T, - participants *ConsensusParticipants, - rootSnapshot protocol.Snapshot, - stopper *Stopper, -) ([]*Node, *Hub) { +// createNodes creates consensus nodes based on the input ConsensusParticipants info. +// All nodes will be started using a common parent context. +// Each node is connected to the Stopper, which will cancel the context when the +// stopping condition is reached. +// The list of created nodes, the common network hub, and a function which starts +// all the nodes together, is returned. +func createNodes(t *testing.T, participants *ConsensusParticipants, rootSnapshot protocol.Snapshot, stopper *Stopper) (nodes []*Node, hub *Hub, runFor func(time.Duration)) { consensus, err := rootSnapshot.Identities(filter.HasRole(flow.RoleConsensus)) require.NoError(t, err) @@ -211,8 +210,8 @@ func createNodes( } }) - hub := NewNetworkHub() - nodes := make([]*Node, 0, len(consensus)) + hub = NewNetworkHub() + nodes = make([]*Node, 0, len(consensus)) for i, identity := range consensus { consensusParticipant := participants.Lookup(identity.NodeID) require.NotNil(t, consensusParticipant) @@ -220,15 +219,32 @@ func createNodes( nodes = append(nodes, node) } - return nodes, hub + // create a context which will be used for all nodes + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx, _ := irrecoverable.WithSignaler(ctx) + + // create a function to return which the test case can use to run the nodes for some maximum duration + // and gracefully stop after. + runFor = func(maxDuration time.Duration) { + runNodes(signalerCtx, nodes) + unittest.RequireCloseBefore(t, stopper.stopped, maxDuration, "expect to get signal from stopper before timeout") + stopNodes(t, cancel, nodes) + } + + stopper.WithStopFunc(func() { + + }) + + return nodes, hub, runFor } func createRootQC(t *testing.T, root *flow.Block, participantData *run.ParticipantData) *flow.QuorumCertificate { consensusCluster := participantData.Identities() votes, err := run.GenerateRootBlockVotes(root, participantData) require.NoError(t, err) - qc, err := run.GenerateRootQC(root, votes, participantData, consensusCluster) + qc, invalidVotes, err := run.GenerateRootQC(root, votes, participantData, consensusCluster) require.NoError(t, err) + require.Len(t, invalidVotes, 0) return qc } @@ -385,7 +401,7 @@ func createNode( Hex("node_id", localID[:]). Logger() - stopConsumer := stopper.AddNode(node) + stopper.AddNode(node) counterConsumer := &CounterConsumer{ finalized: func(total uint) { @@ -396,7 +412,6 @@ func createNode( // log with node index logConsumer := notifications.NewLogConsumer(log) notifier := pubsub.NewDistributor() - notifier.AddConsumer(stopConsumer) notifier.AddConsumer(counterConsumer) notifier.AddConsumer(logConsumer) @@ -439,14 +454,12 @@ func createNode( // selector := filter.HasRole(flow.RoleConsensus) committee, err := committees.NewConsensusCommittee(state, localID) require.NoError(t, err) + consumer.AddConsumer(committee) // initialize the block finalizer final := finalizer.NewFinalizer(db, headersDB, fullState, trace.NewNoopTracer()) - prov := &mocknetwork.Engine{} - prov.On("SubmitLocal", mock.Anything).Return(nil) - - syncCore, err := synccore.New(log, synccore.DefaultConfig(), metricsCollector) + syncCore, err := synccore.New(log, synccore.DefaultConfig(), metricsCollector, rootHeader.ChainID) require.NoError(t, err) qcDistributor := pubsub.NewQCCreatedDistributor() @@ -454,7 +467,7 @@ func createNode( forks, err := consensus.NewForks(rootHeader, headersDB, final, notifier, rootHeader, rootQC) require.NoError(t, err) - validator := consensus.NewValidator(metricsCollector, committee, forks) + validator := consensus.NewValidator(metricsCollector, committee) require.NoError(t, err) keys := &storagemock.SafeBeaconKeys{} @@ -480,46 +493,104 @@ func createNode( persist := persister.New(db, rootHeader.ChainID) - started, err := persist.GetStarted() + livenessData, err := persist.GetLivenessData() require.NoError(t, err) voteProcessorFactory := votecollector.NewCombinedVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) - voteCollectors := voteaggregator.NewVoteCollectors(log, started, workerpool.New(2), createCollectorFactoryMethod) + voteCollectors := voteaggregator.NewVoteCollectors(log, livenessData.CurrentView, workerpool.New(2), createCollectorFactoryMethod) - aggregator, err := voteaggregator.NewVoteAggregator(log, notifier, started, voteCollectors) + voteAggregator, err := voteaggregator.NewVoteAggregator( + log, + metricsCollector, + metricsCollector, + metricsCollector, + notifier, + livenessData.CurrentView, + voteCollectors, + ) + require.NoError(t, err) + + timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutCollectorDistributor.AddConsumer(logConsumer) + + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory( + log, + timeoutCollectorDistributor, + committee, + validator, + msig.ConsensusTimeoutTag, + ) + timeoutCollectorsFactory := timeoutcollector.NewTimeoutCollectorFactory( + log, + notifier, + timeoutCollectorDistributor, + timeoutProcessorFactory, + ) + timeoutCollectors := timeoutaggregator.NewTimeoutCollectors(log, livenessData.CurrentView, timeoutCollectorsFactory) + + timeoutAggregator, err := timeoutaggregator.NewTimeoutAggregator( + log, + metricsCollector, + metricsCollector, + metricsCollector, + notifier, + livenessData.CurrentView, + timeoutCollectors, + ) require.NoError(t, err) hotstuffModules := &consensus.HotstuffModules{ - Forks: forks, - Validator: validator, - Notifier: notifier, - Committee: committee, - Signer: signer, - Persist: persist, - QCCreatedDistributor: qcDistributor, - Aggregator: aggregator, + Forks: forks, + Validator: validator, + Notifier: notifier, + Committee: committee, + Signer: signer, + Persist: persist, + QCCreatedDistributor: qcDistributor, + TimeoutCollectorDistributor: timeoutCollectorDistributor, + VoteAggregator: voteAggregator, + TimeoutAggregator: timeoutAggregator, } + // initialize hotstuff + hot, err := consensus.NewParticipant( + log, + metricsCollector, + build, + rootHeader, + []*flow.Header{}, + hotstuffModules, + consensus.WithMinTimeout(hotstuffTimeout), + func(cfg *consensus.ParticipantConfig) { + cfg.MaxTimeoutObjectRebroadcastInterval = hotstuffTimeout + }, + ) + require.NoError(t, err) + // initialize the compliance engine compCore, err := compliance.NewCore( log, metricsCollector, - tracer, metricsCollector, metricsCollector, + metricsCollector, + tracer, cleaner, headersDB, payloadsDB, fullState, cache, syncCore, - aggregator, + validator, + hot, + voteAggregator, + timeoutAggregator, ) require.NoError(t, err) - comp, err := compliance.NewEngine(log, net, me, prov, compCore) + comp, err := compliance.NewEngine(log, me, compCore) require.NoError(t, err) finalizedHeader, err := synceng.NewFinalizedHeaderCache(log, state, pubsub.NewFinalizationDistributor()) @@ -543,31 +614,38 @@ func createNode( syncCore, finalizedHeader, idProvider, + func(cfg *synceng.Config) { + // use a small pool and scan interval for sync engine + cfg.ScanInterval = 500 * time.Millisecond + cfg.PollInterval = time.Second + }, ) require.NoError(t, err) - // initialize the block finalizer - hot, err := consensus.NewParticipant( + messageHub, err := message_hub.NewMessageHub( log, metricsCollector, - build, + net, + me, comp, - rootHeader, - []*flow.Header{}, - hotstuffModules, - consensus.WithInitialTimeout(hotstuffTimeout), - consensus.WithMinTimeout(hotstuffTimeout), + hot, + voteAggregator, + timeoutAggregator, + state, + payloadsDB, ) - require.NoError(t, err) - comp = comp.WithConsensus(hot) + notifier.AddConsumer(messageHub) node.compliance = comp node.sync = sync node.state = fullState node.hot = hot - node.aggregator = hotstuffModules.Aggregator + node.committee = committee + node.voteAggregator = hotstuffModules.VoteAggregator + node.timeoutAggregator = hotstuffModules.TimeoutAggregator + node.messageHub = messageHub node.headers = headersDB node.net = net node.log = log diff --git a/consensus/integration/slow_test.go b/consensus/integration/slow_test.go index 37b6e46db5a..cff6ced64c3 100644 --- a/consensus/integration/slow_test.go +++ b/consensus/integration/slow_test.go @@ -1,98 +1,133 @@ -//go:build timesensitivetest -// +build timesensitivetest - -// This file includes a few time sensitive tests. They might pass on your powerful local machine -// but fail on slow CI machine. -// For now, these tests are only for local. Run it with the build tag on: -// > go test --tags=relic,timesensitivetest ./... - package integration_test import ( + "sync" "testing" "time" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/network/channels" ) -// verify if a node lost some messages, it's still able to catch up. +// TestMessagesLost verifies if a node lost some messages, it's still able to catch up. func TestMessagesLost(t *testing.T) { + stopper := NewStopper(30, 0) + participantsData := createConsensusIdentities(t, 4) + rootSnapshot := createRootSnapshot(t, participantsData) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) - nodes, stopper, hub := createNodes(t, 5, 1000) + hub.WithFilter(blockNodesFirstMessages(10, nodes[0])) - hub.WithFilter(blockNodesForFirstNMessages(50, nodes[0])) - runNodes(nodes) + runFor(time.Minute) - <-stopper.stopped - - for i := range nodes { - printState(t, nodes, i) - } allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - assertLiveness(t, allViews, 60) + cleanupNodes(nodes) } -// verify if each receiver lost 10% messages, the network can still reach consensus +// TestMessagesLostAcrossNetwork verifies if each receiver lost 10% messages, the network can still reach consensus. func TestMessagesLostAcrossNetwork(t *testing.T) { + stopper := NewStopper(10, 0) + participantsData := createConsensusIdentities(t, 4) + rootSnapshot := createRootSnapshot(t, participantsData) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) - nodes, stopper, hub := createNodes(t, 5, 1500) - - hub.WithFilter(blockReceiverMessagesByPercentage(10)) - runNodes(nodes) + hub.WithFilter(blockReceiverMessagesRandomly(0.1)) - <-stopper.stopped + runFor(time.Minute) - for i := range nodes { - printState(t, nodes, i) - } allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - assertLiveness(t, allViews, 50) + cleanupNodes(nodes) } -// verify if each receiver receive delayed messages, the network can still reach consensus -// the delay might skip some blocks, so should expect to see some gaps in the finalized views -// like this: -// [1 2 3 4 10 11 12 17 20 21 22 23 28 31 33 36 39 44 47 53 58 61 62 79 80 88 89 98 101 106 108 111 115 116 119 120 122 123 124 126 127 128 129 130 133 134 135 138 141 142 143 144] +// TestDelay verifies that we still reach consensus even if _all_ messages are significantly +// delayed. Due to the delay, some proposals might be orphaned. The message delay is sampled +// for each message from the interval [0.1*hotstuffTimeout, 0.5*hotstuffTimeout]. func TestDelay(t *testing.T) { - - nodes, stopper, hub := createNodes(t, 5, 1500) + stopper := NewStopper(30, 0) + participantsData := createConsensusIdentities(t, 4) + rootSnapshot := createRootSnapshot(t, participantsData) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) hub.WithFilter(delayReceiverMessagesByRange(hotstuffTimeout/10, hotstuffTimeout/2)) - runNodes(nodes) - <-stopper.stopped + runFor(time.Minute) - for i := range nodes { - printState(t, nodes, i) - } allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - assertLiveness(t, allViews, 60) + cleanupNodes(nodes) } -// verify that if a node always +// TestOneNodeBehind verifies that if one node (here node 0) consistently experiences a significant +// delay receiving messages beyond the hotstuff timeout, the committee still can reach consensus. func TestOneNodeBehind(t *testing.T) { - nodes, stopper, hub := createNodes(t, 5, 1500) + stopper := NewStopper(30, 0) + participantsData := createConsensusIdentities(t, 3) + rootSnapshot := createRootSnapshot(t, participantsData) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) - hub.WithFilter(func(channelID string, event interface{}, sender, receiver *Node) (bool, time.Duration) { + hub.WithFilter(func(channelID channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { if receiver == nodes[0] { return false, hotstuffTimeout + time.Millisecond } // no block or delay to other nodes return false, 0 }) - runNodes(nodes) - <-stopper.stopped + runFor(time.Minute) + + allViews := allFinalizedViews(t, nodes) + assertSafety(t, allViews) + + cleanupNodes(nodes) +} + +// TestTimeoutRebroadcast drops +// * all proposals at view 5 +// * the first timeout object per view for _every_ sender +// In this configuration, the _initial_ broadcast is insufficient for replicas to make +// progress in view 5 (neither in the happy path, because the proposal is always dropped +// nor on the unhappy path for the _first_ attempt to broadcast timeout objects). We +// expect that replica will eventually broadcast its timeout object again. +func TestTimeoutRebroadcast(t *testing.T) { + stopper := NewStopper(5, 0) + participantsData := createConsensusIdentities(t, 4) + rootSnapshot := createRootSnapshot(t, participantsData) + nodes, hub, runFor := createNodes(t, NewConsensusParticipants(participantsData), rootSnapshot, stopper) + + // nodeID -> view -> numTimeoutMessages + lock := new(sync.Mutex) + blockedTimeoutObjectsTracker := make(map[flow.Identifier]map[uint64]uint64) + hub.WithFilter(func(channelID channels.Channel, event interface{}, sender, receiver *Node) (bool, time.Duration) { + switch m := event.(type) { + case *messages.BlockProposal: + return m.Block.Header.View == 5, 0 // drop proposals only for view 5 + case *messages.TimeoutObject: + // drop first timeout object for every sender for every view + lock.Lock() + blockedPerView, found := blockedTimeoutObjectsTracker[sender.id.NodeID] + if !found { + blockedPerView = make(map[uint64]uint64) + blockedTimeoutObjectsTracker[sender.id.NodeID] = blockedPerView + } + blocked := blockedPerView[m.View] + 1 + blockedPerView[m.View] = blocked + lock.Unlock() + return blocked == 1, 0 + } + // no block or delay to other nodes + return false, 0 + }) + + runFor(10 * time.Second) - for i := range nodes { - printState(t, nodes, i) - } allViews := allFinalizedViews(t, nodes) assertSafety(t, allViews) - assertLiveness(t, allViews, 60) + cleanupNodes(nodes) } diff --git a/consensus/integration/stopper_test.go b/consensus/integration/stopper_test.go index 80e3b32d03e..1c0978ddc18 100644 --- a/consensus/integration/stopper_test.go +++ b/consensus/integration/stopper_test.go @@ -6,35 +6,35 @@ import ( "go.uber.org/atomic" - "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/model/flow" ) -type StopperConsumer struct { - notifications.NoopConsumer -} - +// Stopper is responsible for detecting a stopping condition, and stopping all nodes. +// +// Design motivation: +// - We can stop each node as soon as it enters a certain view. But the problem +// is if some fast node reaches a view earlier and gets stopped, it won't +// be available for other nodes to sync, and slow nodes will never be able +// to catch up. +// - A better strategy is to wait until all nodes have entered a certain view, +// then stop them all - this is what the Stopper does. type Stopper struct { sync.Mutex - running map[flow.Identifier]struct{} - nodes []*Node - stopping *atomic.Bool + running map[flow.Identifier]struct{} + stopping *atomic.Bool + // finalizedCount is the number of blocks which must be finalized (by each node) + // before the stopFunc is called finalizedCount uint - tolerate int - stopped chan struct{} + // tolerate is the number of nodes which we will tolerate NOT finalizing the + // expected number of blocks + tolerate int + stopFunc func() + stopped chan struct{} } -// How to stop nodes? -// We can stop each node as soon as it enters a certain view. But the problem -// is if some fast nodes reaches a view earlier and gets stopped, it won't -// be available for other nodes to sync, and slow nodes will never be able -// to catch up. -// a better strategy is to wait until all nodes has entered a certain view, -// then stop them all. func NewStopper(finalizedCount uint, tolerate int) *Stopper { return &Stopper{ running: make(map[flow.Identifier]struct{}), - nodes: make([]*Node, 0), stopping: atomic.NewBool(false), finalizedCount: finalizedCount, tolerate: tolerate, @@ -42,15 +42,23 @@ func NewStopper(finalizedCount uint, tolerate int) *Stopper { } } -func (s *Stopper) AddNode(n *Node) *StopperConsumer { +// AddNode registers a node with the Stopper, so that the stopping condition is +// adjusted to account for this node (ie. we will now also wait for the added +// node to finalize the desired number of blocks). +func (s *Stopper) AddNode(n *Node) { s.Lock() defer s.Unlock() s.running[n.id.ID()] = struct{}{} - s.nodes = append(s.nodes, n) - stopConsumer := &StopperConsumer{} - return stopConsumer } +// WithStopFunc adds a function to use to stop all nodes (typically the cancel function of the context used to start them). +// Caution: not safe for concurrent use by multiple goroutines. +func (s *Stopper) WithStopFunc(stop func()) { + s.stopFunc = stop +} + +// onFinalizedTotal is called via CounterConsumer each time a node finalizes a block. +// When called, the node with ID `id` has finalized `total` blocks. func (s *Stopper) onFinalizedTotal(id flow.Identifier, total uint) { s.Lock() defer s.Unlock() @@ -69,25 +77,16 @@ func (s *Stopper) onFinalizedTotal(id flow.Identifier, total uint) { } } +// stopAll stops all registered nodes, using the provided stopFunc. func (s *Stopper) stopAll() { - // only allow one process to stop all nodes, - // and stop them exactly once + // only allow one process to stop all nodes, and stop them exactly once if !s.stopping.CompareAndSwap(false, true) { return } - - fmt.Println("stopping all nodes") - - // wait until all nodes has been shut down - var wg sync.WaitGroup - for _, node := range s.nodes { - wg.Add(1) - go func(node *Node) { - node.Shutdown() - wg.Done() - }(node) + if s.stopFunc == nil { + panic("Stopper used without a stopFunc - use WithStopFunc to specify how to stop nodes once stop condition is reached") } - wg.Wait() - + fmt.Println("stopping all nodes...") + s.stopFunc() close(s.stopped) } diff --git a/consensus/participant.go b/consensus/participant.go index a309b9b4a38..9a77f45674a 100644 --- a/consensus/participant.go +++ b/consensus/participant.go @@ -10,15 +10,13 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/eventhandler" "github.com/onflow/flow-go/consensus/hotstuff/eventloop" "github.com/onflow/flow-go/consensus/hotstuff/forks" - "github.com/onflow/flow-go/consensus/hotstuff/forks/finalizer" - "github.com/onflow/flow-go/consensus/hotstuff/forks/forkchoice" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker" "github.com/onflow/flow-go/consensus/hotstuff/pacemaker/timeout" + "github.com/onflow/flow-go/consensus/hotstuff/safetyrules" "github.com/onflow/flow-go/consensus/hotstuff/signature" validatorImpl "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" - "github.com/onflow/flow-go/consensus/hotstuff/voter" "github.com/onflow/flow-go/consensus/recovery" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -30,7 +28,6 @@ func NewParticipant( log zerolog.Logger, metrics module.HotstuffMetrics, builder module.Builder, - communicator hotstuff.Communicator, finalized *flow.Header, pending []*flow.Header, modules *HotstuffModules, @@ -45,35 +42,24 @@ func NewParticipant( option(&cfg) } - // get the last view we started - started, err := modules.Persist.GetStarted() - if err != nil { - return nil, fmt.Errorf("could not recover last started: %w", err) - } - - // get the last view we voted - voted, err := modules.Persist.GetVoted() - if err != nil { - return nil, fmt.Errorf("could not recover last voted: %w", err) - } - // prune vote aggregator to initial view - modules.Aggregator.PruneUpToView(finalized.View) + modules.VoteAggregator.PruneUpToView(finalized.View) + modules.TimeoutAggregator.PruneUpToView(finalized.View) // recover the hotstuff state, mainly to recover all pending blocks in Forks - err = recovery.Participant(log, modules.Forks, modules.Aggregator, modules.Validator, finalized, pending) + err := recovery.Participant(log, modules.Forks, modules.VoteAggregator, modules.Validator, finalized, pending) if err != nil { return nil, fmt.Errorf("could not recover hotstuff state: %w", err) } // initialize the timeout config timeoutConfig, err := timeout.NewConfig( - cfg.TimeoutInitial, cfg.TimeoutMinimum, - cfg.TimeoutAggregationFraction, - cfg.TimeoutIncreaseFactor, - cfg.TimeoutDecreaseFactor, + cfg.TimeoutMaximum, + cfg.TimeoutAdjustmentFactor, + cfg.HappyPathMaxRoundFailures, cfg.BlockRateDelay, + cfg.MaxTimeoutObjectRebroadcastInterval, ) if err != nil { return nil, fmt.Errorf("could not initialize timeout config: %w", err) @@ -81,7 +67,7 @@ func NewParticipant( // initialize the pacemaker controller := timeout.NewController(timeoutConfig) - pacemaker, err := pacemaker.New(started+1, controller, modules.Notifier) + pacemaker, err := pacemaker.New(controller, modules.Notifier, modules.Persist) if err != nil { return nil, fmt.Errorf("could not initialize flow pacemaker: %w", err) } @@ -92,8 +78,11 @@ func NewParticipant( return nil, fmt.Errorf("could not initialize block producer: %w", err) } - // initialize the voter - voter := voter.New(modules.Signer, modules.Forks, modules.Persist, modules.Committee, voted) + // initialize the safetyRules + safetyRules, err := safetyrules.New(modules.Signer, modules.Persist, modules.Committee) + if err != nil { + return nil, fmt.Errorf("could not initialize safety rules: %w", err) + } // initialize the event handler eventHandler, err := eventhandler.NewEventHandler( @@ -102,11 +91,8 @@ func NewParticipant( producer, modules.Forks, modules.Persist, - communicator, modules.Committee, - modules.Aggregator, - voter, - modules.Validator, + safetyRules, modules.Notifier, ) if err != nil { @@ -120,7 +106,8 @@ func NewParticipant( } // add observer, event loop needs to receive events from distributor - modules.QCCreatedDistributor.AddConsumer(loop.SubmitTrustedQC) + modules.QCCreatedDistributor.AddConsumer(loop) + modules.TimeoutCollectorDistributor.AddConsumer(loop) // register dynamically updatable configs if cfg.Registrar != nil { @@ -133,48 +120,31 @@ func NewParticipant( return loop, nil } -// NewForks creates new consensus forks manager -func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.Consumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (hotstuff.Forks, error) { - finalizer, err := newFinalizer(final, headers, updater, notifier, rootHeader, rootQC) - if err != nil { - return nil, fmt.Errorf("could not initialize finalizer: %w", err) - } - - // initialize the fork choice - forkchoice, err := forkchoice.NewNewestForkChoice(finalizer, notifier) - if err != nil { - return nil, fmt.Errorf("could not initialize fork choice: %w", err) - } - - // initialize the Forks manager - return forks.New(finalizer, forkchoice), nil -} - // NewValidator creates new instance of hotstuff validator needed for votes & proposal validation -func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.Committee, forks hotstuff.ForksReader) hotstuff.Validator { +func NewValidator(metrics module.HotstuffMetrics, committee hotstuff.DynamicCommittee) hotstuff.Validator { packer := signature.NewConsensusSigDataPacker(committee) verifier := verification.NewCombinedVerifier(committee, packer) // initialize the Validator - validator := validatorImpl.New(committee, forks, verifier) + validator := validatorImpl.New(committee, verifier) return validatorImpl.NewMetricsWrapper(validator, metrics) // wrapper for measuring time spent in Validator component } -// newFinalizer recovers trusted root and creates new finalizer -func newFinalizer(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*finalizer.Finalizer, error) { +// NewForks recovers trusted root and creates new forks manager +func NewForks(final *flow.Header, headers storage.Headers, updater module.Finalizer, notifier hotstuff.FinalizationConsumer, rootHeader *flow.Header, rootQC *flow.QuorumCertificate) (*forks.Forks, error) { // recover the trusted root trustedRoot, err := recoverTrustedRoot(final, headers, rootHeader, rootQC) if err != nil { return nil, fmt.Errorf("could not recover trusted root: %w", err) } - // initialize the finalizer - finalizer, err := finalizer.New(trustedRoot, updater, notifier) + // initialize the forks + forks, err := forks.New(trustedRoot, updater, notifier) if err != nil { - return nil, fmt.Errorf("could not initialize finalizer: %w", err) + return nil, fmt.Errorf("could not initialize forks: %w", err) } - return finalizer, nil + return forks, nil } // recoverTrustedRoot based on our local state returns root block and QC that can be used to initialize base state @@ -191,12 +161,6 @@ func recoverTrustedRoot(final *flow.Header, headers storage.Headers, rootHeader return makeRootBlockQC(rootHeader, rootQC), nil } - // get the parent for the latest finalized block - parent, err := headers.ByBlockID(final.ParentID) - if err != nil { - return nil, fmt.Errorf("could not get parent for finalized: %w", err) - } - // find a valid child of the finalized block in order to get its QC children, err := headers.ByParentID(final.ID()) if err != nil { @@ -207,11 +171,11 @@ func recoverTrustedRoot(final *flow.Header, headers storage.Headers, rootHeader return nil, fmt.Errorf("finalized block has no children") } - child := model.BlockFromFlow(children[0], final.View) + child := model.BlockFromFlow(children[0]) // create the root block to use trustedRoot := &forks.BlockQC{ - Block: model.BlockFromFlow(final, parent.View), + Block: model.BlockFromFlow(final), QC: child.QC, } diff --git a/consensus/recovery/follower.go b/consensus/recovery/follower.go index 0b8287e3577..c2b3a1e5ed3 100644 --- a/consensus/recovery/follower.go +++ b/consensus/recovery/follower.go @@ -6,26 +6,25 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/forks" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" ) // Follower recovers the HotStuff state for a follower instance. -// It reads the pending blocks from storage and pass them to the input Finalizer +// It reads the pending blocks from storage and pass them to the input Forks // instance to recover its state from before the restart. func Follower( log zerolog.Logger, - finalizer forks.Finalizer, + forks hotstuff.Forks, validator hotstuff.Validator, finalized *flow.Header, pending []*flow.Header, ) error { return Recover(log, finalized, pending, validator, func(proposal *model.Proposal) error { - // add it to finalizer - err := finalizer.AddBlock(proposal.Block) + // add it to forks + err := forks.AddProposal(proposal) if err != nil { - return fmt.Errorf("could not add block to finalizer: %w", err) + return fmt.Errorf("could not add block to forks: %w", err) } log.Debug(). Uint64("block_view", proposal.Block.View). diff --git a/consensus/recovery/participant.go b/consensus/recovery/participant.go index 4ba11c7a293..491349ba9b2 100644 --- a/consensus/recovery/participant.go +++ b/consensus/recovery/participant.go @@ -23,16 +23,13 @@ func Participant( ) error { return Recover(log, finalized, pending, validator, func(proposal *model.Proposal) error { // add it to forks - err := forks.AddBlock(proposal.Block) + err := forks.AddProposal(proposal) if err != nil { return fmt.Errorf("could not add block to forks: %w", err) } - // recovery the proposer's vote - err = voteAggregator.AddBlock(proposal) - if err != nil { - return fmt.Errorf("could not process proposal %v: %w", proposal.Block.BlockID, err) - } + // recover the proposer's vote + voteAggregator.AddBlock(proposal) return nil }) diff --git a/consensus/recovery/recover.go b/consensus/recovery/recover.go index 19514186841..d269f97bfd8 100644 --- a/consensus/recovery/recover.go +++ b/consensus/recovery/recover.go @@ -17,25 +17,13 @@ import ( // received but not finalized, and that share the latest finalized block as a common // ancestor. func Recover(log zerolog.Logger, finalized *flow.Header, pending []*flow.Header, validator hotstuff.Validator, onProposal func(*model.Proposal) error) error { - blocks := make(map[flow.Identifier]*flow.Header, len(pending)+1) - - // finalized is the root - blocks[finalized.ID()] = finalized - log.Info().Int("total", len(pending)).Msgf("recovery started") // add all pending blocks to forks for _, header := range pending { - blocks[header.ID()] = header - - // parent must exist in storage, because the index has the parent ID - parent, ok := blocks[header.ParentID] - if !ok { - return fmt.Errorf("could not find the parent block %x for header %x", header.ParentID, header.ID()) - } // convert the header into a proposal - proposal := model.ProposalFromFlow(header, parent.View) + proposal := model.ProposalFromFlow(header) // verify the proposal err := validator.ValidateProposal(proposal) diff --git a/consensus/vote_aggregator.go b/consensus/vote_aggregator.go deleted file mode 100644 index b388f37a7ec..00000000000 --- a/consensus/vote_aggregator.go +++ /dev/null @@ -1,35 +0,0 @@ -package consensus - -import ( - "fmt" - - "github.com/gammazero/workerpool" - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" - "github.com/onflow/flow-go/consensus/hotstuff/voteaggregator" - "github.com/onflow/flow-go/consensus/hotstuff/votecollector" -) - -// NewVoteAggregator creates new VoteAggregator and recover the Forks' state with all pending block -func NewVoteAggregator( - log zerolog.Logger, - lowestRetainedView uint64, - notifier hotstuff.Consumer, - voteProcessorFactory hotstuff.VoteProcessorFactory, - distributor *pubsub.FinalizationDistributor, -) (hotstuff.VoteAggregator, error) { - - createCollectorFactoryMethod := votecollector.NewStateMachineFactory(log, notifier, voteProcessorFactory.Create) - voteCollectors := voteaggregator.NewVoteCollectors(log, lowestRetainedView, workerpool.New(4), createCollectorFactoryMethod) - - // initialize the vote aggregator - aggregator, err := voteaggregator.NewVoteAggregator(log, notifier, lowestRetainedView, voteCollectors) - if err != nil { - return nil, fmt.Errorf("could not create vote aggregator: %w", err) - } - distributor.AddOnBlockFinalizedConsumer(aggregator.OnFinalizedBlock) - - return aggregator, nil -} diff --git a/crypto/bls.go b/crypto/bls.go index 1d725ebab63..24ac67fcb34 100644 --- a/crypto/bls.go +++ b/crypto/bls.go @@ -31,16 +31,19 @@ package crypto // - membership checks G2 using Bowe's method (https://eprint.iacr.org/2019/814.pdf) // - implement a G1/G2 swap (signatures on G2 and public keys on G1) -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #cgo LDFLAGS: -L${SRCDIR}/relic/build/lib -l relic_s // #include "bls_include.h" import "C" import ( "bytes" + "crypto/sha256" "errors" "fmt" + "golang.org/x/crypto/hkdf" + "github.com/onflow/flow-go/crypto/hash" ) @@ -60,14 +63,14 @@ const ( SignatureLenBLSBLS12381 = fieldSize * (2 - serializationG1) // the length is divided by 2 if compression is on PrKeyLenBLSBLS12381 = 32 // PubKeyLenBLSBLS12381 is the size of G2 elements - PubKeyLenBLSBLS12381 = 2 * fieldSize * (2 - serializationG2) // the length is divided by 2 if compression is on - KeyGenSeedMinLenBLSBLS12381 = PrKeyLenBLSBLS12381 + (securityBits / 8) - KeyGenSeedMaxLenBLSBLS12381 = maxScalarSize + PubKeyLenBLSBLS12381 = 2 * fieldSize * (2 - serializationG2) // the length is divided by 2 if compression is on + + // Hash to curve params // expandMsgOutput is the output length of the expand_message step as required by the hash_to_curve algorithm expandMsgOutput = 2 * (fieldSize + (securityBits / 8)) // hash to curve suite ID of the form : CurveID_ || HashID_ || MapID_ || encodingVariant_ h2cSuiteID = "BLS12381G1_XOF:KMAC128_SSWU_RO_" - // scheme implemented as a countermasure for Rogue attacks of the form : SchemeTag_ + // scheme implemented as a countermasure for rogue attacks of the form : SchemeTag_ schemeTag = "POP_" // Cipher suite used for BLS signatures of the form : BLS_SIG_ || h2cSuiteID || SchemeTag_ blsSigCipherSuite = "BLS_SIG_" + h2cSuiteID + schemeTag @@ -116,10 +119,6 @@ func NewExpandMsgXOFKMAC128(domainTag string) hash.Hasher { // the hash-to-curve function, hashing data to G1 on BLS12 381. // The key is used as a customizer rather than a MAC key. func internalExpandMsgXOFKMAC128(key string) hash.Hasher { - // UTF-8 is used by Go to convert strings into bytes. - // UTF-8 is a non-ambiguous encoding as required by draft-irtf-cfrg-hash-to-curve - // (similarly to the recommended ASCII). - // blsKMACFunction is the customizer used for KMAC in BLS const blsKMACFunction = "H2C" // the error is ignored as the parameter lengths are chosen to be in the correct range for kmac @@ -250,30 +249,67 @@ func IsBLSSignatureIdentity(s Signature) bool { return bytes.Equal(s, identityBLSSignature) } -// generatePrivateKey generates a private key for BLS on BLS12-381 curve. -// The minimum size of the input seed is 48 bytes. +// generatePrivateKey deterministically generates a private key for BLS on BLS12-381 curve. +// The minimum size of the input seed is 32 bytes. // // It is recommended to use a secure crypto RNG to generate the seed. -// The seed must have enough entropy and should be sampled uniformly at random. +// Otherwise, the seed must have enough entropy. // -// The generated private key (resp. its corresponding public key) are guaranteed -// not to be equal to the identity element of Z_r (resp. G2). -func (a *blsBLS12381Algo) generatePrivateKey(seed []byte) (PrivateKey, error) { - if len(seed) < KeyGenSeedMinLenBLSBLS12381 || len(seed) > KeyGenSeedMaxLenBLSBLS12381 { +// The generated private key (resp. its corresponding public key) is guaranteed +// to not be equal to the identity element of Z_r (resp. G2). +func (a *blsBLS12381Algo) generatePrivateKey(ikm []byte) (PrivateKey, error) { + if len(ikm) < KeyGenSeedMinLen || len(ikm) > KeyGenSeedMaxLen { return nil, invalidInputsErrorf( - "seed length should be between %d and %d bytes", - KeyGenSeedMinLenBLSBLS12381, - KeyGenSeedMaxLenBLSBLS12381) + "seed length should be at least %d bytes and at most %d bytes", + KeyGenSeedMinLen, KeyGenSeedMaxLen) } + // HKDF parameters + + // use SHA2-256 as the building block H in HKDF + hashFunction := sha256.New + // salt = H(UTF-8("BLS-SIG-KEYGEN-SALT-")) as per draft-irtf-cfrg-bls-signature-05 section 2.3. + saltString := "BLS-SIG-KEYGEN-SALT-" + hasher := hashFunction() + hasher.Write([]byte(saltString)) + salt := make([]byte, hasher.Size()) + hasher.Sum(salt[:0]) + + // L is the OKM length + // L = ceil((3 * ceil(log2(r))) / 16) which makes L (security_bits/8)-larger than r size + okmLength := (3 * PrKeyLenBLSBLS12381) / 2 + + // HKDF secret = IKM || I2OSP(0, 1) + secret := make([]byte, len(ikm)+1) + copy(secret, ikm) + defer overwrite(secret) // overwrite secret + // HKDF info = key_info || I2OSP(L, 2) + keyInfo := "" // use empty key diversifier. TODO: update header to accept input identifier + info := append([]byte(keyInfo), byte(okmLength>>8), byte(okmLength)) + sk := newPrKeyBLSBLS12381(nil) + for { + // instantiate HKDF and extract L bytes + reader := hkdf.New(hashFunction, secret, salt, info) + okm := make([]byte, okmLength) + n, err := reader.Read(okm) + if err != nil || n != okmLength { + return nil, fmt.Errorf("key generation failed because of the HKDF reader, %d bytes were read: %w", + n, err) + } + defer overwrite(okm) // overwrite okm - // maps the seed to a private key - err := mapToZrStar(&sk.scalar, seed) - if err != nil { - return nil, invalidInputsErrorf("private key generation failed %w", err) + // map the bytes to a private key : SK = OS2IP(OKM) mod r + isZero := mapToZr(&sk.scalar, okm) + if !isZero { + return sk, nil + } + + // update salt = H(salt) + hasher.Reset() + hasher.Write(salt) + salt = hasher.Sum(salt[:0]) } - return sk, nil } const invalidBLSSignatureHeader = byte(0xE0) diff --git a/crypto/bls12381_utils.c b/crypto/bls12381_utils.c index f8af1b0f073..19a1b730b5e 100644 --- a/crypto/bls12381_utils.c +++ b/crypto/bls12381_utils.c @@ -189,36 +189,51 @@ void bn_randZr_star(bn_t x) { byte seed[seed_len]; rand_bytes(seed, seed_len); bn_map_to_Zr_star(x, seed, seed_len); + rand_bytes(seed, seed_len); // overwrite seed } // generates a random number less than the order r void bn_randZr(bn_t x) { - bn_t r; - bn_new(r); - g2_get_ord(r); // reduce the modular reduction bias bn_new_size(x, BITS_TO_DIGITS(Fr_BITS + SEC_BITS)); bn_rand(x, RLC_POS, Fr_BITS + SEC_BITS); - bn_mod(x, x, r); - bn_free(r); + bn_mod(x, x, &core_get()->ep_r); +} + +// Reads a scalar from an array and maps it to Zr. +// The resulting scalar `a` satisfies 0 <= a < r. +// `len` must be less than BITS_TO_BYTES(RLC_BN_BITS). +// It returns VALID if scalar is zero and INVALID otherwise +int bn_map_to_Zr(bn_t a, const uint8_t* bin, int len) { + bn_t tmp; + bn_new(tmp); + bn_new_size(tmp, BYTES_TO_DIGITS(len)); + bn_read_bin(tmp, bin, len); + bn_mod(a, tmp, &core_get()->ep_r); + bn_rand(tmp, RLC_POS, len << 3); // overwrite tmp + bn_free(tmp); + if (bn_cmp_dig(a, 0) == RLC_EQ) { + return VALID; + } + return INVALID; } -// reads a scalar from an array and maps it to Zr -// the resulting scalar is in the range 0 < a < r -// len must be less than BITS_TO_BYTES(RLC_BN_BITS) +// Reads a scalar from an array and maps it to Zr*. +// The resulting scalar `a` satisfies 0 < a < r. +// `len` must be less than BITS_TO_BYTES(RLC_BN_BITS) void bn_map_to_Zr_star(bn_t a, const uint8_t* bin, int len) { bn_t tmp; bn_new(tmp); bn_new_size(tmp, BYTES_TO_DIGITS(len)); bn_read_bin(tmp, bin, len); - bn_t r; - bn_new(r); - g2_get_ord(r); - bn_sub_dig(r,r,1); - bn_mod_basic(a,tmp,r); + bn_t r_1; + bn_new(r_1); + bn_sub_dig(r_1, &core_get()->ep_r, 1); + bn_mod_basic(a,tmp,r_1); bn_add_dig(a,a,1); - bn_free(r); + bn_rand(tmp, RLC_POS, len << 3); // overwrite tmp bn_free(tmp); + bn_free(r_1); } // returns the sign of y. diff --git a/crypto/bls12381_utils.go b/crypto/bls12381_utils.go index 4138d35a599..de11a80ff0e 100644 --- a/crypto/bls12381_utils.go +++ b/crypto/bls12381_utils.go @@ -7,16 +7,17 @@ package crypto // these tools are shared by the BLS signature scheme, the BLS based threshold signature // and the BLS distributed key generation protocols -// #cgo CFLAGS: -g -Wall -std=c99 -I${SRCDIR}/ -I${SRCDIR}/relic/build/include -I${SRCDIR}/relic/include -I${SRCDIR}/relic/include/low +// #cgo CFLAGS: -I${SRCDIR}/ -I${SRCDIR}/relic/build/include -I${SRCDIR}/relic/include -I${SRCDIR}/relic/include/low -I${SRCDIR}/blst_src -I${SRCDIR}/blst_src/build -D__BLST_CGO__ -fno-builtin-memcpy -fno-builtin-memset -Wall -Wno-unused-function -Wno-unused-macros // #cgo LDFLAGS: -L${SRCDIR}/relic/build/lib -l relic_s +// #cgo amd64 CFLAGS: -D__ADX__ -mno-avx +// #cgo mips64 mips64le ppc64 ppc64le riscv64 s390x CFLAGS: -D__BLST_NO_ASM__ // #include "bls12381_utils.h" -// #include "bls_include.h" import "C" import ( "errors" ) -// Go wrappers to Relic C types +// Go wrappers around Relic C types // Relic is compiled with ALLOC=AUTO type pointG1 C.ep_st type pointG2 C.ep2_st @@ -122,18 +123,14 @@ func randZrStar(x *scalar) { C.bn_randZr_star((*C.bn_st)(x)) } -// mapToZrStar reads a scalar from a slice of bytes and maps it to Zr -// the resulting scalar is in the range 0 < k < r -func mapToZrStar(x *scalar, src []byte) error { - if len(src) > maxScalarSize { - return invalidInputsErrorf( - "input slice length must be less than %d", - maxScalarSize) - } - C.bn_map_to_Zr_star((*C.bn_st)(x), +// mapToZr reads a scalar from a slice of bytes and maps it to Zr. +// The resulting scalar `k` satisfies 0 <= k < r. +// It returns true if scalar is zero and false otherwise. +func mapToZr(x *scalar, src []byte) bool { + isZero := C.bn_map_to_Zr((*C.bn_st)(x), (*C.uchar)(&src[0]), (C.int)(len(src))) - return nil + return isZero == valid } // writeScalar writes a G2 point in a slice of bytes diff --git a/crypto/bls12381_utils.h b/crypto/bls12381_utils.h index c7e3587f664..c6d8e48b3b4 100644 --- a/crypto/bls12381_utils.h +++ b/crypto/bls12381_utils.h @@ -8,8 +8,7 @@ #define _REL_MISC_INCLUDE_H #include "relic.h" - -typedef uint8_t byte; +#include "blst_include.h" #define VALID RLC_OK #define INVALID RLC_ERR @@ -106,6 +105,7 @@ void ep2_mult_gen(ep2_t, const bn_t); void bn_randZr(bn_t); void bn_randZr_star(bn_t); +int bn_map_to_Zr(bn_t, const uint8_t*, int); void bn_map_to_Zr_star(bn_t, const uint8_t*, int); void bn_sum_vector(bn_t, const bn_st*, const int); diff --git a/crypto/bls12381_utils_test.go b/crypto/bls12381_utils_test.go index 8911ada1769..074268f25b8 100644 --- a/crypto/bls12381_utils_test.go +++ b/crypto/bls12381_utils_test.go @@ -14,9 +14,9 @@ import ( func TestDeterministicKeyGen(t *testing.T) { // 2 keys generated with the same seed should be equal - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) n, err := rand.Read(seed) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk1, err := GeneratePrivateKey(BLSBLS12381, seed) require.Nil(t, err) @@ -29,9 +29,9 @@ func TestDeterministicKeyGen(t *testing.T) { func TestPRGseeding(t *testing.T) { blsInstance.reInit() // 2 scalars generated with the same seed should be equal - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) n, err := rand.Read(seed) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) // 1st scalar (wrapped in a private key) err = seedRelic(seed) diff --git a/crypto/bls_crossBLST_test.go b/crypto/bls_crossBLST_test.go index f2ef6f16431..143454dfb25 100644 --- a/crypto/bls_crossBLST_test.go +++ b/crypto/bls_crossBLST_test.go @@ -16,18 +16,25 @@ package crypto // both libraries might have made different choices. It is nevertheless a good flag for possible bugs or deviations // from the standard as both libraries are being developed. -import ( +/*import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" blst "github.com/supranational/blst/bindings/go" "pgregory.net/rapid" -) + + "github.com/onflow/flow-go/crypto" +)*/ + +// TODO: this file can't compile because of duplicate C and assembly symbols (the ones used +// by the current library and the same ones used by the imported package BLST). Unfortunately, +// cgo doesn't differentiate the two symbols. These tests need to be rewritten using the internal +// BLST C exports, instead of importing the Go BLST package. // validPrivateKeyBytesFlow generates bytes of a valid private key in Flow library -func validPrivateKeyBytesFlow(t *rapid.T) []byte { - seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381).Draw(t, "seed").([]byte) +/*func validPrivateKeyBytesFlow(t *rapid.T) []byte { + seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "seed").([]byte) sk, err := GeneratePrivateKey(BLSBLS12381, seed) // TODO: require.NoError(t, err) seems to mess with rapid if err != nil { @@ -38,7 +45,7 @@ func validPrivateKeyBytesFlow(t *rapid.T) []byte { // validPublicKeyBytesFlow generates bytes of a valid public key in Flow library func validPublicKeyBytesFlow(t *rapid.T) []byte { - seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381).Draw(t, "seed").([]byte) + seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "seed").([]byte) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) return sk.PublicKey().Encode() @@ -46,10 +53,10 @@ func validPublicKeyBytesFlow(t *rapid.T) []byte { // validSignatureBytesFlow generates bytes of a valid signature in Flow library func validSignatureBytesFlow(t *rapid.T) []byte { - seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381).Draw(t, "seed").([]byte) + seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "seed").([]byte) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) - hasher := NewExpandMsgXOFKMAC128("random_tag") + hasher := crypto.NewExpandMsgXOFKMAC128("random_tag") message := rapid.SliceOfN(rapid.Byte(), 1, 1000).Draw(t, "msg").([]byte) signature, err := sk.Sign(message, hasher) require.NoError(t, err) @@ -58,14 +65,14 @@ func validSignatureBytesFlow(t *rapid.T) []byte { // validPrivateKeyBytesBLST generates bytes of a valid private key in BLST library func validPrivateKeyBytesBLST(t *rapid.T) []byte { - randomSlice := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381) + randomSlice := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen) ikm := randomSlice.Draw(t, "ikm").([]byte) return blst.KeyGen(ikm).Serialize() } // validPublicKeyBytesBLST generates bytes of a valid public key in BLST library func validPublicKeyBytesBLST(t *rapid.T) []byte { - ikm := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381).Draw(t, "ikm").([]byte) + ikm := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "ikm").([]byte) blstS := blst.KeyGen(ikm) blstG2 := new(blst.P2Affine).From(blstS) return blstG2.Compress() @@ -73,7 +80,7 @@ func validPublicKeyBytesBLST(t *rapid.T) []byte { // validSignatureBytesBLST generates bytes of a valid signature in BLST library func validSignatureBytesBLST(t *rapid.T) []byte { - ikm := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381).Draw(t, "ikm").([]byte) + ikm := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "ikm").([]byte) blstS := blst.KeyGen(ikm[:]) blstG1 := new(blst.P1Affine).From(blstS) return blstG1.Compress() @@ -82,14 +89,14 @@ func validSignatureBytesBLST(t *rapid.T) []byte { // testEncodeDecodePrivateKeyCrossBLST tests encoding and decoding of private keys are consistent with BLST. // This test assumes private key serialization is identical to the one in BLST. func testEncodeDecodePrivateKeyCrossBLST(t *rapid.T) { - randomSlice := rapid.SliceOfN(rapid.Byte(), prKeyLengthBLSBLS12381, prKeyLengthBLSBLS12381) + randomSlice := rapid.SliceOfN(rapid.Byte(), crypto.PrKeyLenBLSBLS12381, crypto.PrKeyLenBLSBLS12381) validSliceFlow := rapid.Custom(validPrivateKeyBytesFlow) validSliceBLST := rapid.Custom(validPrivateKeyBytesBLST) // skBytes are bytes of either a valid or a random private key skBytes := rapid.OneOf(randomSlice, validSliceFlow, validSliceBLST).Example().([]byte) // check decoding results are consistent - skFlow, err := DecodePrivateKey(BLSBLS12381, skBytes) + skFlow, err := crypto.DecodePrivateKey(crypto.BLSBLS12381, skBytes) var skBLST blst.Scalar res := skBLST.Deserialize(skBytes) @@ -109,14 +116,14 @@ func testEncodeDecodePrivateKeyCrossBLST(t *rapid.T) { // testEncodeDecodePublicKeyCrossBLST tests encoding and decoding of public keys keys are consistent with BLST. // This test assumes public key serialization is identical to the one in BLST. func testEncodeDecodePublicKeyCrossBLST(t *rapid.T) { - randomSlice := rapid.SliceOfN(rapid.Byte(), PubKeyLenBLSBLS12381, PubKeyLenBLSBLS12381) + randomSlice := rapid.SliceOfN(rapid.Byte(), crypto.PubKeyLenBLSBLS12381, crypto.PubKeyLenBLSBLS12381) validSliceFlow := rapid.Custom(validPublicKeyBytesFlow) validSliceBLST := rapid.Custom(validPublicKeyBytesBLST) // pkBytes are bytes of either a valid or a random public key pkBytes := rapid.OneOf(randomSlice, validSliceFlow, validSliceBLST).Example().([]byte) // check decoding results are consistent - pkFlow, err := DecodePublicKey(BLSBLS12381, pkBytes) + pkFlow, err := crypto.DecodePublicKey(crypto.BLSBLS12381, pkBytes) var pkBLST blst.P2Affine res := pkBLST.Deserialize(pkBytes) pkValidBLST := pkBLST.KeyValidate() @@ -137,7 +144,7 @@ func testEncodeDecodePublicKeyCrossBLST(t *rapid.T) { // testEncodeDecodeSignatureCrossBLST tests encoding and decoding of signatures are consistent with BLST. // This test assumes signature serialization is identical to the one in BLST. func testEncodeDecodeSignatureCrossBLST(t *rapid.T) { - randomSlice := rapid.SliceOfN(rapid.Byte(), SignatureLenBLSBLS12381, SignatureLenBLSBLS12381) + randomSlice := rapid.SliceOfN(rapid.Byte(), crypto.SignatureLenBLSBLS12381, crypto.SignatureLenBLSBLS12381) validSignatureFlow := rapid.Custom(validSignatureBytesFlow) validSignatureBLST := rapid.Custom(validSignatureBytesBLST) // sigBytes are bytes of either a valid or a random signature @@ -180,7 +187,7 @@ func testSignHashCrossBLST(t *rapid.T) { // generate two private keys from the same seed skBytes := rapid.Custom(validPrivateKeyBytesFlow).Example().([]byte) - skFlow, err := DecodePrivateKey(BLSBLS12381, skBytes) + skFlow, err := crypto.DecodePrivateKey(crypto.BLSBLS12381, skBytes) require.NoError(t, err) var skBLST blst.Scalar res := skBLST.Deserialize(skBytes) @@ -203,9 +210,21 @@ func testSignHashCrossBLST(t *rapid.T) { assert.Equal(t, sigBytesBLST, sigBytesFlow) } +func testKeyGenCrossBLST(t *rapid.T) { + seed := rapid.SliceOfN(rapid.Byte(), KeyGenSeedMinLen, KeyGenSeedMaxLen).Draw(t, "seed").([]byte) + + skFlow, err := GeneratePrivateKey(BLSBLS12381, seed) + if err != nil { + assert.FailNow(t, "failed key generation") + } + skBLST := blst.KeyGen(seed) + assert.Equal(t, skFlow.Encode(), skBLST.Serialize()) +} + func TestAgainstBLST(t *testing.T) { + rapid.Check(t, testKeyGenCrossBLST) rapid.Check(t, testEncodeDecodePrivateKeyCrossBLST) rapid.Check(t, testEncodeDecodePublicKeyCrossBLST) rapid.Check(t, testEncodeDecodeSignatureCrossBLST) rapid.Check(t, testSignHashCrossBLST) -} +}*/ diff --git a/crypto/bls_multisig.go b/crypto/bls_multisig.go index 1dfe29abc05..a915bed4a64 100644 --- a/crypto/bls_multisig.go +++ b/crypto/bls_multisig.go @@ -27,7 +27,7 @@ import ( // - batch verification of multiple signatures of a single message under multiple // public keys: use a binary tree of aggregations to find the invalid signatures. -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #cgo LDFLAGS: -L${SRCDIR}/relic/build/lib -l relic_s // #include "bls_include.h" import "C" diff --git a/crypto/bls_no_relic.go b/crypto/bls_no_relic.go index ce69b3bbd84..fed6c216398 100644 --- a/crypto/bls_no_relic.go +++ b/crypto/bls_no_relic.go @@ -18,8 +18,7 @@ import ( const relic_panic = "function is not supported when building without \"relic\" Go build tag" const ( - SignatureLenBLSBLS12381 = 48 - KeyGenSeedMinLenBLSBLS12381 = 48 + SignatureLenBLSBLS12381 = 48 ) // bls.go functions diff --git a/crypto/bls_test.go b/crypto/bls_test.go index d0dc73c066c..c674137d8e3 100644 --- a/crypto/bls_test.go +++ b/crypto/bls_test.go @@ -19,10 +19,10 @@ import ( // TestBLSMainMethods is a sanity check of main signature scheme methods (keyGen, sign, verify) func TestBLSMainMethods(t *testing.T) { - hasher := NewExpandMsgXOFKMAC128("test tag") // test the key generation seed lengths - testKeyGenSeed(t, BLSBLS12381, KeyGenSeedMinLenBLSBLS12381, KeyGenSeedMaxLenBLSBLS12381) + testKeyGenSeed(t, BLSBLS12381, KeyGenSeedMinLen, KeyGenSeedMaxLen) // test the consistency with different inputs + hasher := NewExpandMsgXOFKMAC128("test tag") testGenSignVerify(t, BLSBLS12381, hasher) // specific signature test for BLS: @@ -51,7 +51,6 @@ func TestBLSMainMethods(t *testing.T) { require.NoError(t, err) assert.False(t, valid) }) - } // Signing bench @@ -69,7 +68,7 @@ func BenchmarkBLSBLS12381Verify(b *testing.B) { // utility function to generate a random BLS private key func randomSK(t *testing.T, seed []byte) PrivateKey { n, err := rand.Read(seed) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) @@ -78,9 +77,9 @@ func randomSK(t *testing.T, seed []byte) PrivateKey { // utility function to generate a non BLS private key func invalidSK(t *testing.T) PrivateKey { - seed := make([]byte, KeyGenSeedMinLenECDSAP256) + seed := make([]byte, KeyGenSeedMinLen) n, err := rand.Read(seed) - require.Equal(t, n, KeyGenSeedMinLenECDSAP256) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk, err := GeneratePrivateKey(ECDSAP256, seed) require.NoError(t, err) @@ -90,7 +89,7 @@ func invalidSK(t *testing.T) PrivateKey { // BLS tests func TestBLSBLS12381Hasher(t *testing.T) { // generate a key pair - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) sk := randomSK(t, seed) sig := make([]byte, SignatureLenBLSBLS12381) @@ -207,7 +206,7 @@ func TestBLSEquals(t *testing.T) { // TestBLSUtils tests some utility functions func TestBLSUtils(t *testing.T) { // generate a key pair - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) sk := randomSK(t, seed) // test Algorithm() testKeysAlgorithm(t, sk, BLSBLS12381) @@ -290,7 +289,7 @@ func TestBLSAggregateSignatures(t *testing.T) { sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) var aggSig, expectedSig Signature // create the signatures @@ -421,7 +420,7 @@ func TestBLSAggregatePubKeys(t *testing.T) { pkNum := mrand.Intn(100) + 1 pks := make([]PublicKey, 0, pkNum) sks := make([]PrivateKey, 0, pkNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) // create the signatures for i := 0; i < pkNum; i++ { @@ -515,7 +514,7 @@ func TestBLSRemovePubKeys(t *testing.T) { // number of keys to aggregate pkNum := mrand.Intn(100) + 1 pks := make([]PublicKey, 0, pkNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) // generate public keys for i := 0; i < pkNum; i++ { @@ -622,7 +621,7 @@ func TestBLSBatchVerify(t *testing.T) { sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) expectedValid := make([]bool, 0, sigsNum) // create the signatures @@ -763,7 +762,7 @@ func BenchmarkBatchVerify(b *testing.B) { sigsNum := 100 sigs := make([]Signature, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) // create the signatures for i := 0; i < sigsNum; i++ { @@ -824,7 +823,7 @@ func TestBLSAggregateSignaturesManyMessages(t *testing.T) { // number of keys keysNum := mrand.Intn(sigsNum) + 1 sks := make([]PrivateKey, 0, keysNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) // generate the keys for i := 0; i < keysNum; i++ { sk := randomSK(t, seed) @@ -972,7 +971,7 @@ func BenchmarkVerifySignatureManyMessages(b *testing.B) { inputKmacs := make([]hash.Hasher, 0, sigsNum) sigs := make([]Signature, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) inputMsgs := make([][]byte, 0, sigsNum) kmac := NewExpandMsgXOFKMAC128("bench tag") @@ -1010,7 +1009,7 @@ func BenchmarkAggregate(b *testing.B) { sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) // create the signatures for i := 0; i < sigsNum; i++ { @@ -1073,7 +1072,7 @@ func TestBLSIdentity(t *testing.T) { assert.True(t, IsBLSSignatureIdentity(identityBLSSignature)) // sum up a random signature and its inverse to get identity - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) sk := randomSK(t, seed) sig, err := sk.Sign(msg, hasher) require.NoError(t, err) diff --git a/crypto/bls_thresholdsign.go b/crypto/bls_thresholdsign.go index 4256af84ab9..4aa73278d3a 100644 --- a/crypto/bls_thresholdsign.go +++ b/crypto/bls_thresholdsign.go @@ -3,7 +3,7 @@ package crypto -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #include "bls_thresholdsign_include.h" import "C" diff --git a/crypto/bls_thresholdsign_test.go b/crypto/bls_thresholdsign_test.go index cc9be81eeaf..947de451987 100644 --- a/crypto/bls_thresholdsign_test.go +++ b/crypto/bls_thresholdsign_test.go @@ -268,7 +268,7 @@ func testCentralizedStatefulAPI(t *testing.T) { assert.True(t, IsInvalidInputsError(err)) assert.Nil(t, tsFollower) // non BLS key share - seed := make([]byte, KeyGenSeedMinLenECDSAP256) + seed := make([]byte, KeyGenSeedMinLen) _, err = rand.Read(seed) require.NoError(t, err) skEcdsa, err := GeneratePrivateKey(ECDSAP256, seed) diff --git a/crypto/blst_assembly.S b/crypto/blst_assembly.S new file mode 100644 index 00000000000..a1a7c5416e0 --- /dev/null +++ b/crypto/blst_assembly.S @@ -0,0 +1,123 @@ +#if defined(__x86_64) || defined(__x86_64__) +# if defined(__ELF__) +# if defined(__BLST_PORTABLE__) +# include "elf/sha256-portable-x86_64.s" +# else +# include "elf/sha256-x86_64.s" +# endif +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "elf/ctx_inverse_mod_384-x86_64.s" +# else +# include "elf/ctq_inverse_mod_384-x86_64.s" +# endif +# include "elf/add_mod_384-x86_64.s" +# include "elf/add_mod_384x384-x86_64.s" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# define __sub_mod_384x384 __sub_mont_384x384 +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "elf/mulx_mont_384-x86_64.s" +# include "elf/mulx_mont_256-x86_64.s" +# else +# include "elf/mulq_mont_384-x86_64.s" +# include "elf/mulq_mont_256-x86_64.s" +# endif +# include "elf/add_mod_256-x86_64.s" +# include "elf/ct_inverse_mod_256-x86_64.s" +# include "elf/div3w-x86_64.s" +# include "elf/ct_is_square_mod_384-x86_64.s" +# elif defined(_WIN64) || defined(__CYGWIN__) +# if defined(__BLST_PORTABLE__) +# include "coff/sha256-portable-x86_64.s" +# else +# include "coff/sha256-x86_64.s" +# endif +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "coff/ctx_inverse_mod_384-x86_64.s" +# else +# include "coff/ctq_inverse_mod_384-x86_64.s" +# endif +# include "coff/add_mod_384-x86_64.s" +# include "coff/add_mod_384x384-x86_64.s" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# define __sub_mod_384x384 __sub_mont_384x384 +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "coff/mulx_mont_384-x86_64.s" +# include "coff/mulx_mont_256-x86_64.s" +# else +# include "coff/mulq_mont_384-x86_64.s" +# include "coff/mulq_mont_256-x86_64.s" +# endif +# include "coff/add_mod_256-x86_64.s" +# include "coff/ct_inverse_mod_256-x86_64.s" +# include "coff/div3w-x86_64.s" +# include "coff/ct_is_square_mod_384-x86_64.s" +# elif defined(__APPLE__) +# include "mach-o/sha256-x86_64.s" +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "mach-o/ctx_inverse_mod_384-x86_64.s" +# else +# include "mach-o/ctq_inverse_mod_384-x86_64.s" +# endif +# include "mach-o/add_mod_384-x86_64.s" +# include "mach-o/add_mod_384x384-x86_64.s" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# define __sub_mod_384x384 __sub_mont_384x384 +# if defined(__ADX__) && !defined(__BLST_PORTABLE__) +# include "mach-o/mulx_mont_384-x86_64.s" +# include "mach-o/mulx_mont_256-x86_64.s" +# else +# include "mach-o/mulq_mont_384-x86_64.s" +# include "mach-o/mulq_mont_256-x86_64.s" +# endif +# include "mach-o/add_mod_256-x86_64.s" +# include "mach-o/ct_inverse_mod_256-x86_64.s" +# include "mach-o/div3w-x86_64.s" +# include "mach-o/ct_is_square_mod_384-x86_64.s" +# endif +#elif defined(__aarch64__) +# if defined(__ELF__) +# include "elf/sha256-armv8.S" +# include "elf/ct_inverse_mod_384-armv8.S" +# include "elf/add_mod_384-armv8.S" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# include "elf/mul_mont_384-armv8.S" +# include "elf/mul_mont_256-armv8.S" +# include "elf/add_mod_256-armv8.S" +# include "elf/ct_inverse_mod_256-armv8.S" +# include "elf/div3w-armv8.S" +# include "elf/ct_is_square_mod_384-armv8.S" +# elif defined(_WIN64) +# include "coff/sha256-armv8.S" +# include "coff/ct_inverse_mod_384-armv8.S" +# include "coff/add_mod_384-armv8.S" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# include "coff/mul_mont_384-armv8.S" +# include "coff/mul_mont_256-armv8.S" +# include "coff/add_mod_256-armv8.S" +# include "coff/ct_inverse_mod_256-armv8.S" +# include "coff/div3w-armv8.S" +# include "coff/ct_is_square_mod_384-armv8.S" +# elif defined(__APPLE__) +# include "mach-o/sha256-armv8.S" +# include "mach-o/ct_inverse_mod_384-armv8.S" +# include "mach-o/add_mod_384-armv8.S" +# define __add_mod_384 __add_mont_384 +# define __sub_mod_384 __sub_mont_384 +# include "mach-o/mul_mont_384-armv8.S" +# include "mach-o/mul_mont_256-armv8.S" +# include "mach-o/add_mod_256-armv8.S" +# include "mach-o/ct_inverse_mod_256-armv8.S" +# include "mach-o/div3w-armv8.S" +# include "mach-o/ct_is_square_mod_384-armv8.S" +# endif +#elif defined(__BLST_NO_ASM__) || \ + (defined(__SIZEOF_POINTER__) && __SIZEOF_POINTER__==4) +/* inaccurate way to detect a 32-bit processor, but it's close enough */ +#else +# error "unsupported platform" +#endif diff --git a/crypto/blst_include.h b/crypto/blst_include.h new file mode 100644 index 00000000000..dde3acd5f05 --- /dev/null +++ b/crypto/blst_include.h @@ -0,0 +1,11 @@ +// +build relic + +#ifndef __BLST_INCLUDE_H__ +#define __BLST_INCLUDE_H__ + +// blst related definitions +// eventually this file would replace blst.h + +#include "blst.h" + +#endif \ No newline at end of file diff --git a/crypto/blst_src.c b/crypto/blst_src.c new file mode 100644 index 00000000000..89388b703fe --- /dev/null +++ b/crypto/blst_src.c @@ -0,0 +1,19 @@ +// +build relic + +#include "keygen.c" +#include "hash_to_field.c" +#include "e1.c" +#include "map_to_g1.c" +#include "e2.c" +#include "map_to_g2.c" +#include "fp12_tower.c" +#include "pairing.c" +#include "aggregate.c" +#include "exp.c" +#include "sqrt.c" +#include "recip.c" +#include "bulk_addition.c" +#include "multi_scalar.c" +#include "consts.c" +#include "vect.c" +#include "exports.c" diff --git a/crypto/blst_src/LICENSE b/crypto/blst_src/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/crypto/blst_src/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crypto/blst_src/README.md b/crypto/blst_src/README.md new file mode 100644 index 00000000000..12bc7b863ca --- /dev/null +++ b/crypto/blst_src/README.md @@ -0,0 +1,16 @@ +All files in this folder contain source files copied from the BLST repo https://github.com/supranational/blst +specifically from the commit <92c12ac58095de04e776cec5ef5ce5bdf242b693>. + + Copyright Supranational LLC + Licensed under the Apache License, Version 2.0, see LICENSE for details. + SPDX-License-Identifier: Apache-2.0 + +While BLST exports multiple functions and tools, the implementation in Flow crypto requires access to low level functions. Some of these tools are not exported by BLST, others would need to be used without paying for the cgo cost, and therefore without using the Go bindings in BLST. + + +The folder contains: +- BLST LICENSE file +- all <blst>/src/*.c and <blst>/src/*.h files (C source files) +- all <blst>/build (assembly generated files) +- <blst>/bindings/blst.h (headers of external functions) +- <blst>/bindings/blst_aux.h (headers of external aux functions) \ No newline at end of file diff --git a/crypto/blst_src/aggregate.c b/crypto/blst_src/aggregate.c new file mode 100644 index 00000000000..8a24e0590ba --- /dev/null +++ b/crypto/blst_src/aggregate.c @@ -0,0 +1,673 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * Usage pattern on single-processor system is + * + * blst_pairing_init(ctx, hash_or_encode, DST); + * blst_pairing_aggregate_pk_in_g1(ctx, PK[0], aggregated_signature, msg[0]); + * blst_pairing_aggregate_pk_in_g1(ctx, PK[1], NULL, msg[1]); + * ... + * blst_pairing_commit(ctx); + * blst_pairing_finalverify(ctx, NULL); + * + *********************************************************************** + * Usage pattern on multi-processor system is + * + * blst_pairing_init(pk[0], hash_or_encode, DST); + * blst_pairing_init(pk[1], hash_or_encode, DST); + * ... + * start threads each processing an N/nthreads slice of PKs and messages: + * blst_pairing_aggregate_pk_in_g1(pk[i], PK[i*n+0], NULL, msg[i*n+0]); + * blst_pairing_aggregate_pk_in_g1(pk[i], PK[i*n+1], NULL, msg[i*n+1]); + * ... + * blst_pairing_commit(pkx); + * ... + * meanwhile in main thread + * blst_fp12 gtsig; + * blst_aggregated_in_g2(>sig, aggregated_signature); + * join threads and merge their contexts: + * blst_pairing_merge(pk[0], pk[1]); + * blst_pairing_merge(pk[0], pk[2]); + * ... + * blst_pairing_finalverify(pk[0], gtsig); + */ + +#ifndef N_MAX +# define N_MAX 8 +#endif + +typedef union { POINTonE1 e1; POINTonE2 e2; } AggregatedSignature; +typedef struct { + unsigned int ctrl; + unsigned int nelems; + const void *DST; + size_t DST_len; + vec384fp12 GT; + AggregatedSignature AggrSign; + POINTonE2_affine Q[N_MAX]; + POINTonE1_affine P[N_MAX]; +} PAIRING; + +enum { AGGR_UNDEFINED = 0, + AGGR_MIN_SIG = 1, + AGGR_MIN_PK = 2, + AGGR_SIGN_SET = 0x10, + AGGR_GT_SET = 0x20, + AGGR_HASH_OR_ENCODE = 0x40 }; +#define MIN_SIG_OR_PK (AGGR_MIN_SIG | AGGR_MIN_PK) + +static const size_t sizeof_pairing = (sizeof(PAIRING) + 7) & ~(size_t)7; + +size_t blst_pairing_sizeof(void) +{ return sizeof_pairing; } + +void blst_pairing_init(PAIRING *ctx, int hash_or_encode, + const void *DST, size_t DST_len) +{ + ctx->ctrl = AGGR_UNDEFINED | (hash_or_encode ? AGGR_HASH_OR_ENCODE : 0); + ctx->nelems = 0; + ctx->DST = (uptr_t)DST==(uptr_t)((byte *)ctx+sizeof_pairing) ? (void *)42 + : DST; + ctx->DST_len = DST_len; +} + +static const void *pairing_get_dst(const PAIRING *ctx) +{ return (uptr_t)ctx->DST==(uptr_t)42 ? (const byte *)ctx+sizeof_pairing + : ctx->DST; +} + +const void *blst_pairing_get_dst(const PAIRING *ctx) +{ return pairing_get_dst(ctx); } + +#define FROM_AFFINE(out,in) do { \ + vec_copy((out)->X, in->X, 2*sizeof(in->X)), \ + vec_select((out)->Z, in->X, BLS12_381_Rx.p, sizeof(in->X), \ + vec_is_zero(in->X, 2*sizeof(in->X))); } while(0) + +/* + * Optional |nbits|-wide |scalar| is used to facilitate multiple aggregated + * signature vetification as discussed at + * https://ethresear.ch/t/fast-verification-of-multiple-bls-signatures/5407. + * Usage pattern is not finalized yet, because (sig != NULL) is better and + * will be handled separately... + */ +static BLST_ERROR PAIRING_Aggregate_PK_in_G2(PAIRING *ctx, + const POINTonE2_affine *PK, + size_t pk_groupcheck, + const POINTonE1_affine *sig, + size_t sig_groupcheck, + const byte *scalar, size_t nbits, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ + if (ctx->ctrl & AGGR_MIN_PK) + return BLST_AGGR_TYPE_MISMATCH; + + ctx->ctrl |= AGGR_MIN_SIG; + + /* + * Since we don't know if the signature is individual or aggregated, + * the only sensible thing to do is to skip over infinite one and + * count on the corresponding infinite public key to be rejected, + * in case the signature is non-aggregated that is. + */ + if (sig != NULL && !vec_is_zero(sig, sizeof(*sig))) { + POINTonE1 *S = &ctx->AggrSign.e1; + POINTonE1 P[1]; + + FROM_AFFINE(P, sig); + + if (sig_groupcheck && !POINTonE1_in_G1(P)) + return BLST_POINT_NOT_IN_GROUP; + + if (ctx->ctrl & AGGR_SIGN_SET) { + if (nbits != 0 && scalar != NULL) { + POINTonE1_mult_w5(P, P, scalar, nbits); + POINTonE1_dadd(S, S, P, NULL); + } else { + POINTonE1_dadd_affine(S, S, sig); + } + } else { + ctx->ctrl |= AGGR_SIGN_SET; + if (nbits != 0 && scalar != NULL) + POINTonE1_mult_w5(S, P, scalar, nbits); + else + vec_copy(S, P, sizeof(P)); + } + } + + if (PK != NULL) { + unsigned int n; + POINTonE1 H[1]; + const void *DST = pairing_get_dst(ctx); + + /* + * Reject infinite public keys. + */ + if (vec_is_zero(PK, sizeof(*PK))) + return BLST_PK_IS_INFINITY; + + if (pk_groupcheck) { + POINTonE2 P[1]; + + FROM_AFFINE(P, PK); + if (!POINTonE2_in_G2(P)) + return BLST_POINT_NOT_IN_GROUP; + } + + if (ctx->ctrl & AGGR_HASH_OR_ENCODE) + Hash_to_G1(H, msg, msg_len, DST, ctx->DST_len, aug, aug_len); + else + Encode_to_G1(H, msg, msg_len, DST, ctx->DST_len, aug, aug_len); + + if (nbits != 0 && scalar != NULL) + POINTonE1_mult_w5(H, H, scalar, nbits); + + POINTonE1_from_Jacobian(H, H); + + n = ctx->nelems; + vec_copy(ctx->Q + n, PK, sizeof(POINTonE2_affine)); + vec_copy(ctx->P + n, H, sizeof(POINTonE1_affine)); + if (++n == N_MAX) { + if (ctx->ctrl & AGGR_GT_SET) { + vec384fp12 GT; + miller_loop_n(GT, ctx->Q, ctx->P, n); + mul_fp12(ctx->GT, ctx->GT, GT); + } else { + miller_loop_n(ctx->GT, ctx->Q, ctx->P, n); + ctx->ctrl |= AGGR_GT_SET; + } + n = 0; + } + ctx->nelems = n; + } + + return BLST_SUCCESS; +} + +BLST_ERROR blst_pairing_aggregate_pk_in_g2(PAIRING *ctx, + const POINTonE2_affine *PK, + const POINTonE1_affine *signature, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G2(ctx, PK, 0, signature, 1, NULL, 0, + msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_mul_n_aggregate_pk_in_g2(PAIRING *ctx, + const POINTonE2_affine *PK, + const POINTonE1_affine *sig, + const byte *scalar, + size_t nbits, + const void *msg, + size_t msg_len, + const void *aug, + size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G2(ctx, PK, 0, sig, 1, scalar, nbits, + msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_chk_n_aggr_pk_in_g2(PAIRING *ctx, + const POINTonE2_affine *PK, + size_t pk_grpchk, + const POINTonE1_affine *signature, + size_t sig_grpchk, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G2(ctx, PK, pk_grpchk, signature, sig_grpchk, + NULL, 0, msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_chk_n_mul_n_aggr_pk_in_g2(PAIRING *ctx, + const POINTonE2_affine *PK, + size_t pk_grpchk, + const POINTonE1_affine *sig, + size_t sig_grpchk, + const byte *scalar, + size_t nbits, + const void *msg, + size_t msg_len, + const void *aug, + size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G2(ctx, PK, pk_grpchk, sig, sig_grpchk, + scalar, nbits, + msg, msg_len, aug, aug_len); +} + +static BLST_ERROR PAIRING_Aggregate_PK_in_G1(PAIRING *ctx, + const POINTonE1_affine *PK, + size_t pk_groupcheck, + const POINTonE2_affine *sig, + size_t sig_groupcheck, + const byte *scalar, size_t nbits, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ + if (ctx->ctrl & AGGR_MIN_SIG) + return BLST_AGGR_TYPE_MISMATCH; + + ctx->ctrl |= AGGR_MIN_PK; + + /* + * Since we don't know if the signature is individual or aggregated, + * the only sensible thing to do is to skip over infinite one and + * count on the corresponding infinite public key to be rejected, + * in case the signature is non-aggregated that is. + */ + if (sig != NULL && !vec_is_zero(sig, sizeof(*sig))) { + POINTonE2 *S = &ctx->AggrSign.e2; + POINTonE2 P[1]; + + FROM_AFFINE(P, sig); + + if (sig_groupcheck && !POINTonE2_in_G2(P)) + return BLST_POINT_NOT_IN_GROUP; + + if (ctx->ctrl & AGGR_SIGN_SET) { + if (nbits != 0 && scalar != NULL) { + + POINTonE2_mult_w5(P, P, scalar, nbits); + POINTonE2_dadd(S, S, P, NULL); + } else { + POINTonE2_dadd_affine(S, S, sig); + } + } else { + ctx->ctrl |= AGGR_SIGN_SET; + if (nbits != 0 && scalar != NULL) + POINTonE2_mult_w5(S, P, scalar, nbits); + else + vec_copy(S, P, sizeof(P)); + } + } + + if (PK != NULL) { + unsigned int n; + POINTonE2 H[1]; + POINTonE1 pk[1]; + const void *DST = pairing_get_dst(ctx); + + /* + * Reject infinite public keys. + */ + if (vec_is_zero(PK, sizeof(*PK))) + return BLST_PK_IS_INFINITY; + + if (pk_groupcheck) { + POINTonE1 P[1]; + + FROM_AFFINE(P, PK); + if (!POINTonE1_in_G1(P)) + return BLST_POINT_NOT_IN_GROUP; + } + + if (ctx->ctrl & AGGR_HASH_OR_ENCODE) + Hash_to_G2(H, msg, msg_len, DST, ctx->DST_len, aug, aug_len); + else + Encode_to_G2(H, msg, msg_len, DST, ctx->DST_len, aug, aug_len); + + POINTonE2_from_Jacobian(H, H); + + if (nbits != 0 && scalar != NULL) { + FROM_AFFINE(pk, PK); + POINTonE1_mult_w5(pk, pk, scalar, nbits); + POINTonE1_from_Jacobian(pk, pk); + PK = (const POINTonE1_affine *)pk; + } + + n = ctx->nelems; + vec_copy(ctx->Q + n, H, sizeof(POINTonE2_affine)); + vec_copy(ctx->P + n, PK, sizeof(POINTonE1_affine)); + if (++n == N_MAX) { + if (ctx->ctrl & AGGR_GT_SET) { + vec384fp12 GT; + miller_loop_n(GT, ctx->Q, ctx->P, n); + mul_fp12(ctx->GT, ctx->GT, GT); + } else { + miller_loop_n(ctx->GT, ctx->Q, ctx->P, n); + ctx->ctrl |= AGGR_GT_SET; + } + n = 0; + } + ctx->nelems = n; + } + + return BLST_SUCCESS; +} + +BLST_ERROR blst_pairing_aggregate_pk_in_g1(PAIRING *ctx, + const POINTonE1_affine *PK, + const POINTonE2_affine *signature, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G1(ctx, PK, 0, signature, 1, NULL, 0, + msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_mul_n_aggregate_pk_in_g1(PAIRING *ctx, + const POINTonE1_affine *PK, + const POINTonE2_affine *sig, + const byte *scalar, + size_t nbits, + const void *msg, + size_t msg_len, + const void *aug, + size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G1(ctx, PK, 0, sig, 1, scalar, nbits, + msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_chk_n_aggr_pk_in_g1(PAIRING *ctx, + const POINTonE1_affine *PK, + size_t pk_grpchk, + const POINTonE2_affine *signature, + size_t sig_grpchk, + const void *msg, size_t msg_len, + const void *aug, size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G1(ctx, PK, pk_grpchk, signature, sig_grpchk, + NULL, 0, msg, msg_len, aug, aug_len); +} + +BLST_ERROR blst_pairing_chk_n_mul_n_aggr_pk_in_g1(PAIRING *ctx, + const POINTonE1_affine *PK, + size_t pk_grpchk, + const POINTonE2_affine *sig, + size_t sig_grpchk, + const byte *scalar, + size_t nbits, + const void *msg, + size_t msg_len, + const void *aug, + size_t aug_len) +{ return PAIRING_Aggregate_PK_in_G1(ctx, PK, pk_grpchk, sig, sig_grpchk, + scalar, nbits, + msg, msg_len, aug, aug_len); +} + +static void PAIRING_Commit(PAIRING *ctx) +{ + unsigned int n; + + if ((n = ctx->nelems) != 0) { + if (ctx->ctrl & AGGR_GT_SET) { + vec384fp12 GT; + miller_loop_n(GT, ctx->Q, ctx->P, n); + mul_fp12(ctx->GT, ctx->GT, GT); + } else { + miller_loop_n(ctx->GT, ctx->Q, ctx->P, n); + ctx->ctrl |= AGGR_GT_SET; + } + ctx->nelems = 0; + } +} + +void blst_pairing_commit(PAIRING *ctx) +{ PAIRING_Commit(ctx); } + +BLST_ERROR blst_pairing_merge(PAIRING *ctx, const PAIRING *ctx1) +{ + if ((ctx->ctrl & MIN_SIG_OR_PK) != AGGR_UNDEFINED + && (ctx1->ctrl & MIN_SIG_OR_PK) != AGGR_UNDEFINED + && (ctx->ctrl & ctx1->ctrl & MIN_SIG_OR_PK) == 0) + return BLST_AGGR_TYPE_MISMATCH; + + /* context producers are expected to have called blst_pairing_commit */ + if (ctx->nelems || ctx1->nelems) + return BLST_AGGR_TYPE_MISMATCH; + + ctx->ctrl |= ctx1->ctrl & MIN_SIG_OR_PK; + + switch (ctx->ctrl & MIN_SIG_OR_PK) { + case AGGR_MIN_SIG: + if (ctx->ctrl & ctx1->ctrl & AGGR_SIGN_SET) { + POINTonE1_dadd(&ctx->AggrSign.e1, &ctx->AggrSign.e1, + &ctx1->AggrSign.e1, NULL); + } else if (ctx1->ctrl & AGGR_SIGN_SET) { + ctx->ctrl |= AGGR_SIGN_SET; + vec_copy(&ctx->AggrSign.e1, &ctx1->AggrSign.e1, + sizeof(ctx->AggrSign.e1)); + } + break; + case AGGR_MIN_PK: + if (ctx->ctrl & ctx1->ctrl & AGGR_SIGN_SET) { + POINTonE2_dadd(&ctx->AggrSign.e2, &ctx->AggrSign.e2, + &ctx1->AggrSign.e2, NULL); + } else if (ctx1->ctrl & AGGR_SIGN_SET) { + ctx->ctrl |= AGGR_SIGN_SET; + vec_copy(&ctx->AggrSign.e2, &ctx1->AggrSign.e2, + sizeof(ctx->AggrSign.e2)); + } + break; + case AGGR_UNDEFINED: + break; + default: + return BLST_AGGR_TYPE_MISMATCH; + } + + if (ctx->ctrl & ctx1->ctrl & AGGR_GT_SET) { + mul_fp12(ctx->GT, ctx->GT, ctx1->GT); + } else if (ctx1->ctrl & AGGR_GT_SET) { + ctx->ctrl |= AGGR_GT_SET; + vec_copy(ctx->GT, ctx1->GT, sizeof(ctx->GT)); + } + + return BLST_SUCCESS; +} + +static bool_t PAIRING_FinalVerify(const PAIRING *ctx, const vec384fp12 GTsig) +{ + vec384fp12 GT; + + if (!(ctx->ctrl & AGGR_GT_SET)) + return 0; + + if (GTsig != NULL) { + vec_copy(GT, GTsig, sizeof(GT)); + } else if (ctx->ctrl & AGGR_SIGN_SET) { + AggregatedSignature AggrSign; + + switch (ctx->ctrl & MIN_SIG_OR_PK) { + case AGGR_MIN_SIG: + POINTonE1_from_Jacobian(&AggrSign.e1, &ctx->AggrSign.e1); + miller_loop_n(GT, (const POINTonE2_affine *)&BLS12_381_G2, + (const POINTonE1_affine *)&AggrSign.e1, 1); + break; + case AGGR_MIN_PK: + POINTonE2_from_Jacobian(&AggrSign.e2, &ctx->AggrSign.e2); + miller_loop_n(GT, (const POINTonE2_affine *)&AggrSign.e2, + (const POINTonE1_affine *)&BLS12_381_G1, 1); + break; + default: + return 0; + } + } else { + /* + * The aggregated signature was infinite, relation between the + * hashes and the public keys has to be VERY special... + */ + vec_copy(GT, BLS12_381_Rx.p12, sizeof(GT)); + } + + conjugate_fp12(GT); + mul_fp12(GT, GT, ctx->GT); + final_exp(GT, GT); + + /* return GT==1 */ + return vec_is_equal(GT[0][0], BLS12_381_Rx.p2, sizeof(GT[0][0])) & + vec_is_zero(GT[0][1], sizeof(GT) - sizeof(GT[0][0])); +} + +int blst_pairing_finalverify(const PAIRING *ctx, const vec384fp12 GTsig) +{ return (int)PAIRING_FinalVerify(ctx, GTsig); } + +int blst_fp12_finalverify(const vec384fp12 GT1, const vec384fp12 GT2) +{ + vec384fp12 GT; + + vec_copy(GT, GT1, sizeof(GT)); + conjugate_fp12(GT); + mul_fp12(GT, GT, GT2); + final_exp(GT, GT); + + /* return GT==1 */ + return (int)(vec_is_equal(GT[0][0], BLS12_381_Rx.p2, sizeof(GT[0][0])) & + vec_is_zero(GT[0][1], sizeof(GT) - sizeof(GT[0][0]))); +} + +void blst_pairing_raw_aggregate(PAIRING *ctx, const POINTonE2_affine *q, + const POINTonE1_affine *p) +{ + unsigned int n; + + if (vec_is_zero(q, sizeof(*q)) & vec_is_zero(p, sizeof(*p))) + return; + + n = ctx->nelems; + vec_copy(ctx->Q + n, q, sizeof(*q)); + vec_copy(ctx->P + n, p, sizeof(*p)); + if (++n == N_MAX) { + if (ctx->ctrl & AGGR_GT_SET) { + vec384fp12 GT; + miller_loop_n(GT, ctx->Q, ctx->P, n); + mul_fp12(ctx->GT, ctx->GT, GT); + } else { + miller_loop_n(ctx->GT, ctx->Q, ctx->P, n); + ctx->ctrl |= AGGR_GT_SET; + } + n = 0; + } + ctx->nelems = n; +} + +vec384fp12 *blst_pairing_as_fp12(PAIRING *ctx) +{ + PAIRING_Commit(ctx); + return (vec384fp12 *)ctx->GT; +} + +/* + * PAIRING context-free entry points. + * + * To perform FastAggregateVerify, aggregate all public keys and + * signatures with corresponding blst_aggregate_in_g{12}, convert + * result to affine and call suitable blst_core_verify_pk_in_g{12} + * or blst_aggregated_in_g{12}... + */ +BLST_ERROR blst_aggregate_in_g1(POINTonE1 *out, const POINTonE1 *in, + const unsigned char *zwire) +{ + POINTonE1 P[1]; + BLST_ERROR ret; + + ret = POINTonE1_Deserialize_Z((POINTonE1_affine *)P, zwire); + + if (ret != BLST_SUCCESS) + return ret; + + if (vec_is_zero(P, sizeof(POINTonE1_affine))) { + if (in == NULL) + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } + + vec_copy(P->Z, BLS12_381_Rx.p, sizeof(P->Z)); + + if (!POINTonE1_in_G1(P)) + return BLST_POINT_NOT_IN_GROUP; + + if (in == NULL) + vec_copy(out, P, sizeof(P)); + else + POINTonE1_dadd_affine(out, in, (POINTonE1_affine *)P); + + return BLST_SUCCESS; +} + +BLST_ERROR blst_aggregate_in_g2(POINTonE2 *out, const POINTonE2 *in, + const unsigned char *zwire) +{ + POINTonE2 P[1]; + BLST_ERROR ret; + + ret = POINTonE2_Deserialize_Z((POINTonE2_affine *)P, zwire); + + if (ret != BLST_SUCCESS) + return ret; + + if (vec_is_zero(P, sizeof(POINTonE2_affine))) { + if (in == NULL) + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } + + vec_copy(P->Z, BLS12_381_Rx.p, sizeof(P->Z)); + + if (!POINTonE2_in_G2(P)) + return BLST_POINT_NOT_IN_GROUP; + + if (in == NULL) { + vec_copy(out, P, sizeof(P)); + } else { + POINTonE2_dadd_affine(out, in, (POINTonE2_affine *)P); + } + return BLST_SUCCESS; +} + +void blst_aggregated_in_g1(vec384fp12 ret, const POINTonE1_affine *sig) +{ miller_loop_n(ret, (const POINTonE2_affine *)&BLS12_381_G2, sig, 1); } + +void blst_aggregated_in_g2(vec384fp12 ret, const POINTonE2_affine *sig) +{ miller_loop_n(ret, sig, (const POINTonE1_affine *)&BLS12_381_G1, 1); } + +BLST_ERROR blst_core_verify_pk_in_g1(const POINTonE1_affine *pk, + const POINTonE2_affine *signature, + int hash_or_encode, + const void *msg, size_t msg_len, + const void *DST, size_t DST_len, + const void *aug, size_t aug_len) +{ + PAIRING ctx; + BLST_ERROR ret; + + ctx.ctrl = AGGR_UNDEFINED | (hash_or_encode ? AGGR_HASH_OR_ENCODE : 0); + ctx.nelems = 0; + ctx.DST = DST; + ctx.DST_len = DST_len; + + ret = PAIRING_Aggregate_PK_in_G1(&ctx, pk, 1, signature, 1, NULL, 0, + msg, msg_len, aug, aug_len); + if (ret != BLST_SUCCESS) + return ret; + + PAIRING_Commit(&ctx); + + return PAIRING_FinalVerify(&ctx, NULL) ? BLST_SUCCESS : BLST_VERIFY_FAIL; +} + +BLST_ERROR blst_core_verify_pk_in_g2(const POINTonE2_affine *pk, + const POINTonE1_affine *signature, + int hash_or_encode, + const void *msg, size_t msg_len, + const void *DST, size_t DST_len, + const void *aug, size_t aug_len) +{ + PAIRING ctx; + BLST_ERROR ret; + + ctx.ctrl = AGGR_UNDEFINED | (hash_or_encode ? AGGR_HASH_OR_ENCODE : 0); + ctx.nelems = 0; + ctx.DST = DST; + ctx.DST_len = DST_len; + + ret = PAIRING_Aggregate_PK_in_G2(&ctx, pk, 1, signature, 1, NULL, 0, + msg, msg_len, aug, aug_len); + if (ret != BLST_SUCCESS) + return ret; + + PAIRING_Commit(&ctx); + + return PAIRING_FinalVerify(&ctx, NULL) ? BLST_SUCCESS : BLST_VERIFY_FAIL; +} diff --git a/crypto/blst_src/blst.h b/crypto/blst_src/blst.h new file mode 100644 index 00000000000..24213ded2c5 --- /dev/null +++ b/crypto/blst_src/blst.h @@ -0,0 +1,483 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLST_H__ +#define __BLST_H__ + +#ifdef __SIZE_TYPE__ +typedef __SIZE_TYPE__ size_t; +#else +#include <stddef.h> +#endif + +#if defined(__UINT8_TYPE__) && defined(__UINT32_TYPE__) \ + && defined(__UINT64_TYPE__) +typedef __UINT8_TYPE__ uint8_t; +typedef __UINT32_TYPE__ uint32_t; +typedef __UINT64_TYPE__ uint64_t; +#else +#include <stdint.h> +#endif + +#ifdef __cplusplus +extern "C" { +#elif defined(__BLST_CGO__) +typedef _Bool bool; /* it's assumed that cgo calls modern enough compiler */ +#elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 +# define bool _Bool +#else +# define bool int +#endif + +#ifdef SWIG +# define DEFNULL =NULL +#elif defined __cplusplus +# define DEFNULL =0 +#else +# define DEFNULL +#endif + +typedef enum { + BLST_SUCCESS = 0, + BLST_BAD_ENCODING, + BLST_POINT_NOT_ON_CURVE, + BLST_POINT_NOT_IN_GROUP, + BLST_AGGR_TYPE_MISMATCH, + BLST_VERIFY_FAIL, + BLST_PK_IS_INFINITY, + BLST_BAD_SCALAR, +} BLST_ERROR; + +typedef uint8_t byte; +typedef uint64_t limb_t; + +typedef struct { byte b[256/8]; } blst_scalar; +typedef struct { limb_t l[256/8/sizeof(limb_t)]; } blst_fr; +typedef struct { limb_t l[384/8/sizeof(limb_t)]; } blst_fp; +/* 0 is "real" part, 1 is "imaginary" */ +typedef struct { blst_fp fp[2]; } blst_fp2; +typedef struct { blst_fp2 fp2[3]; } blst_fp6; +typedef struct { blst_fp6 fp6[2]; } blst_fp12; + +void blst_scalar_from_uint32(blst_scalar *out, const uint32_t a[8]); +void blst_uint32_from_scalar(uint32_t out[8], const blst_scalar *a); +void blst_scalar_from_uint64(blst_scalar *out, const uint64_t a[4]); +void blst_uint64_from_scalar(uint64_t out[4], const blst_scalar *a); +void blst_scalar_from_bendian(blst_scalar *out, const byte a[32]); +void blst_bendian_from_scalar(byte out[32], const blst_scalar *a); +void blst_scalar_from_lendian(blst_scalar *out, const byte a[32]); +void blst_lendian_from_scalar(byte out[32], const blst_scalar *a); +bool blst_scalar_fr_check(const blst_scalar *a); +bool blst_sk_check(const blst_scalar *a); +bool blst_sk_add_n_check(blst_scalar *out, const blst_scalar *a, + const blst_scalar *b); +bool blst_sk_sub_n_check(blst_scalar *out, const blst_scalar *a, + const blst_scalar *b); +bool blst_sk_mul_n_check(blst_scalar *out, const blst_scalar *a, + const blst_scalar *b); +void blst_sk_inverse(blst_scalar *out, const blst_scalar *a); +bool blst_scalar_from_le_bytes(blst_scalar *out, const byte *in, size_t len); +bool blst_scalar_from_be_bytes(blst_scalar *out, const byte *in, size_t len); + +#ifndef SWIG +/* + * BLS12-381-specifc Fr operations. + */ +void blst_fr_add(blst_fr *ret, const blst_fr *a, const blst_fr *b); +void blst_fr_sub(blst_fr *ret, const blst_fr *a, const blst_fr *b); +void blst_fr_mul_by_3(blst_fr *ret, const blst_fr *a); +void blst_fr_lshift(blst_fr *ret, const blst_fr *a, size_t count); +void blst_fr_rshift(blst_fr *ret, const blst_fr *a, size_t count); +void blst_fr_mul(blst_fr *ret, const blst_fr *a, const blst_fr *b); +void blst_fr_sqr(blst_fr *ret, const blst_fr *a); +void blst_fr_cneg(blst_fr *ret, const blst_fr *a, bool flag); +void blst_fr_eucl_inverse(blst_fr *ret, const blst_fr *a); +void blst_fr_inverse(blst_fr *ret, const blst_fr *a); +#ifdef BLST_FR_PENTAROOT +void blst_fr_pentaroot(blst_fr *ret, const blst_fr *a); +void blst_fr_pentapow(blst_fr *ret, const blst_fr *a); +#endif + +void blst_fr_from_uint64(blst_fr *ret, const uint64_t a[4]); +void blst_uint64_from_fr(uint64_t ret[4], const blst_fr *a); +void blst_fr_from_scalar(blst_fr *ret, const blst_scalar *a); +void blst_scalar_from_fr(blst_scalar *ret, const blst_fr *a); + +/* + * BLS12-381-specifc Fp operations. + */ +void blst_fp_add(blst_fp *ret, const blst_fp *a, const blst_fp *b); +void blst_fp_sub(blst_fp *ret, const blst_fp *a, const blst_fp *b); +void blst_fp_mul_by_3(blst_fp *ret, const blst_fp *a); +void blst_fp_mul_by_8(blst_fp *ret, const blst_fp *a); +void blst_fp_lshift(blst_fp *ret, const blst_fp *a, size_t count); +void blst_fp_mul(blst_fp *ret, const blst_fp *a, const blst_fp *b); +void blst_fp_sqr(blst_fp *ret, const blst_fp *a); +void blst_fp_cneg(blst_fp *ret, const blst_fp *a, bool flag); +void blst_fp_eucl_inverse(blst_fp *ret, const blst_fp *a); +void blst_fp_inverse(blst_fp *ret, const blst_fp *a); +bool blst_fp_sqrt(blst_fp *ret, const blst_fp *a); + +void blst_fp_from_uint32(blst_fp *ret, const uint32_t a[12]); +void blst_uint32_from_fp(uint32_t ret[12], const blst_fp *a); +void blst_fp_from_uint64(blst_fp *ret, const uint64_t a[6]); +void blst_uint64_from_fp(uint64_t ret[6], const blst_fp *a); +void blst_fp_from_bendian(blst_fp *ret, const byte a[48]); +void blst_bendian_from_fp(byte ret[48], const blst_fp *a); +void blst_fp_from_lendian(blst_fp *ret, const byte a[48]); +void blst_lendian_from_fp(byte ret[48], const blst_fp *a); + +/* + * BLS12-381-specifc Fp2 operations. + */ +void blst_fp2_add(blst_fp2 *ret, const blst_fp2 *a, const blst_fp2 *b); +void blst_fp2_sub(blst_fp2 *ret, const blst_fp2 *a, const blst_fp2 *b); +void blst_fp2_mul_by_3(blst_fp2 *ret, const blst_fp2 *a); +void blst_fp2_mul_by_8(blst_fp2 *ret, const blst_fp2 *a); +void blst_fp2_lshift(blst_fp2 *ret, const blst_fp2 *a, size_t count); +void blst_fp2_mul(blst_fp2 *ret, const blst_fp2 *a, const blst_fp2 *b); +void blst_fp2_sqr(blst_fp2 *ret, const blst_fp2 *a); +void blst_fp2_cneg(blst_fp2 *ret, const blst_fp2 *a, bool flag); +void blst_fp2_eucl_inverse(blst_fp2 *ret, const blst_fp2 *a); +void blst_fp2_inverse(blst_fp2 *ret, const blst_fp2 *a); +bool blst_fp2_sqrt(blst_fp2 *ret, const blst_fp2 *a); + +/* + * BLS12-381-specifc Fp12 operations. + */ +void blst_fp12_sqr(blst_fp12 *ret, const blst_fp12 *a); +void blst_fp12_cyclotomic_sqr(blst_fp12 *ret, const blst_fp12 *a); +void blst_fp12_mul(blst_fp12 *ret, const blst_fp12 *a, const blst_fp12 *b); +void blst_fp12_mul_by_xy00z0(blst_fp12 *ret, const blst_fp12 *a, + const blst_fp6 *xy00z0); +void blst_fp12_conjugate(blst_fp12 *a); +void blst_fp12_inverse(blst_fp12 *ret, const blst_fp12 *a); +/* caveat lector! |n| has to be non-zero and not more than 3! */ +void blst_fp12_frobenius_map(blst_fp12 *ret, const blst_fp12 *a, size_t n); +bool blst_fp12_is_equal(const blst_fp12 *a, const blst_fp12 *b); +bool blst_fp12_is_one(const blst_fp12 *a); +bool blst_fp12_in_group(const blst_fp12 *a); +const blst_fp12 *blst_fp12_one(); +#endif // SWIG + +/* + * BLS12-381-specifc point operations. + */ +typedef struct { blst_fp x, y, z; } blst_p1; +typedef struct { blst_fp x, y; } blst_p1_affine; + +void blst_p1_add(blst_p1 *out, const blst_p1 *a, const blst_p1 *b); +void blst_p1_add_or_double(blst_p1 *out, const blst_p1 *a, const blst_p1 *b); +void blst_p1_add_affine(blst_p1 *out, const blst_p1 *a, + const blst_p1_affine *b); +void blst_p1_add_or_double_affine(blst_p1 *out, const blst_p1 *a, + const blst_p1_affine *b); +void blst_p1_double(blst_p1 *out, const blst_p1 *a); +void blst_p1_mult(blst_p1 *out, const blst_p1 *p, const byte *scalar, + size_t nbits); +void blst_p1_cneg(blst_p1 *p, bool cbit); +void blst_p1_to_affine(blst_p1_affine *out, const blst_p1 *in); +void blst_p1_from_affine(blst_p1 *out, const blst_p1_affine *in); +bool blst_p1_on_curve(const blst_p1 *p); +bool blst_p1_in_g1(const blst_p1 *p); +bool blst_p1_is_equal(const blst_p1 *a, const blst_p1 *b); +bool blst_p1_is_inf(const blst_p1 *a); +const blst_p1 *blst_p1_generator(); + +bool blst_p1_affine_on_curve(const blst_p1_affine *p); +bool blst_p1_affine_in_g1(const blst_p1_affine *p); +bool blst_p1_affine_is_equal(const blst_p1_affine *a, const blst_p1_affine *b); +bool blst_p1_affine_is_inf(const blst_p1_affine *a); +const blst_p1_affine *blst_p1_affine_generator(); + +typedef struct { blst_fp2 x, y, z; } blst_p2; +typedef struct { blst_fp2 x, y; } blst_p2_affine; + +void blst_p2_add(blst_p2 *out, const blst_p2 *a, const blst_p2 *b); +void blst_p2_add_or_double(blst_p2 *out, const blst_p2 *a, const blst_p2 *b); +void blst_p2_add_affine(blst_p2 *out, const blst_p2 *a, + const blst_p2_affine *b); +void blst_p2_add_or_double_affine(blst_p2 *out, const blst_p2 *a, + const blst_p2_affine *b); +void blst_p2_double(blst_p2 *out, const blst_p2 *a); +void blst_p2_mult(blst_p2 *out, const blst_p2 *p, const byte *scalar, + size_t nbits); +void blst_p2_cneg(blst_p2 *p, bool cbit); +void blst_p2_to_affine(blst_p2_affine *out, const blst_p2 *in); +void blst_p2_from_affine(blst_p2 *out, const blst_p2_affine *in); +bool blst_p2_on_curve(const blst_p2 *p); +bool blst_p2_in_g2(const blst_p2 *p); +bool blst_p2_is_equal(const blst_p2 *a, const blst_p2 *b); +bool blst_p2_is_inf(const blst_p2 *a); +const blst_p2 *blst_p2_generator(); + +bool blst_p2_affine_on_curve(const blst_p2_affine *p); +bool blst_p2_affine_in_g2(const blst_p2_affine *p); +bool blst_p2_affine_is_equal(const blst_p2_affine *a, const blst_p2_affine *b); +bool blst_p2_affine_is_inf(const blst_p2_affine *a); +const blst_p2_affine *blst_p2_affine_generator(); + +/* + * Multi-scalar multiplications and other multi-point operations. + */ + +void blst_p1s_to_affine(blst_p1_affine dst[], const blst_p1 *const points[], + size_t npoints); +void blst_p1s_add(blst_p1 *ret, const blst_p1_affine *const points[], + size_t npoints); + +size_t blst_p1s_mult_wbits_precompute_sizeof(size_t wbits, size_t npoints); +void blst_p1s_mult_wbits_precompute(blst_p1_affine table[], size_t wbits, + const blst_p1_affine *const points[], + size_t npoints); +size_t blst_p1s_mult_wbits_scratch_sizeof(size_t npoints); +void blst_p1s_mult_wbits(blst_p1 *ret, const blst_p1_affine table[], + size_t wbits, size_t npoints, + const byte *const scalars[], size_t nbits, + limb_t *scratch); + +size_t blst_p1s_mult_pippenger_scratch_sizeof(size_t npoints); +void blst_p1s_mult_pippenger(blst_p1 *ret, const blst_p1_affine *const points[], + size_t npoints, const byte *const scalars[], + size_t nbits, limb_t *scratch); +void blst_p1s_tile_pippenger(blst_p1 *ret, const blst_p1_affine *const points[], + size_t npoints, const byte *const scalars[], + size_t nbits, limb_t *scratch, + size_t bit0, size_t window); + +void blst_p2s_to_affine(blst_p2_affine dst[], const blst_p2 *const points[], + size_t npoints); +void blst_p2s_add(blst_p2 *ret, const blst_p2_affine *const points[], + size_t npoints); + +size_t blst_p2s_mult_wbits_precompute_sizeof(size_t wbits, size_t npoints); +void blst_p2s_mult_wbits_precompute(blst_p2_affine table[], size_t wbits, + const blst_p2_affine *const points[], + size_t npoints); +size_t blst_p2s_mult_wbits_scratch_sizeof(size_t npoints); +void blst_p2s_mult_wbits(blst_p2 *ret, const blst_p2_affine table[], + size_t wbits, size_t npoints, + const byte *const scalars[], size_t nbits, + limb_t *scratch); + +size_t blst_p2s_mult_pippenger_scratch_sizeof(size_t npoints); +void blst_p2s_mult_pippenger(blst_p2 *ret, const blst_p2_affine *const points[], + size_t npoints, const byte *const scalars[], + size_t nbits, limb_t *scratch); +void blst_p2s_tile_pippenger(blst_p2 *ret, const blst_p2_affine *const points[], + size_t npoints, const byte *const scalars[], + size_t nbits, limb_t *scratch, + size_t bit0, size_t window); + +/* + * Hash-to-curve operations. + */ +#ifndef SWIG +void blst_map_to_g1(blst_p1 *out, const blst_fp *u, const blst_fp *v DEFNULL); +void blst_map_to_g2(blst_p2 *out, const blst_fp2 *u, const blst_fp2 *v DEFNULL); +#endif + +void blst_encode_to_g1(blst_p1 *out, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, size_t DST_len DEFNULL, + const byte *aug DEFNULL, size_t aug_len DEFNULL); +void blst_hash_to_g1(blst_p1 *out, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, size_t DST_len DEFNULL, + const byte *aug DEFNULL, size_t aug_len DEFNULL); + +void blst_encode_to_g2(blst_p2 *out, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, size_t DST_len DEFNULL, + const byte *aug DEFNULL, size_t aug_len DEFNULL); +void blst_hash_to_g2(blst_p2 *out, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, size_t DST_len DEFNULL, + const byte *aug DEFNULL, size_t aug_len DEFNULL); + +/* + * Zcash-compatible serialization/deserialization. + */ +void blst_p1_serialize(byte out[96], const blst_p1 *in); +void blst_p1_compress(byte out[48], const blst_p1 *in); +void blst_p1_affine_serialize(byte out[96], const blst_p1_affine *in); +void blst_p1_affine_compress(byte out[48], const blst_p1_affine *in); +BLST_ERROR blst_p1_uncompress(blst_p1_affine *out, const byte in[48]); +BLST_ERROR blst_p1_deserialize(blst_p1_affine *out, const byte in[96]); + +void blst_p2_serialize(byte out[192], const blst_p2 *in); +void blst_p2_compress(byte out[96], const blst_p2 *in); +void blst_p2_affine_serialize(byte out[192], const blst_p2_affine *in); +void blst_p2_affine_compress(byte out[96], const blst_p2_affine *in); +BLST_ERROR blst_p2_uncompress(blst_p2_affine *out, const byte in[96]); +BLST_ERROR blst_p2_deserialize(blst_p2_affine *out, const byte in[192]); + +/* + * Specification defines two variants, 'minimal-signature-size' and + * 'minimal-pubkey-size'. To unify appearance we choose to distinguish + * them by suffix referring to the public key type, more specifically + * _pk_in_g1 corresponds to 'minimal-pubkey-size' and _pk_in_g2 - to + * 'minimal-signature-size'. It might appear a bit counterintuitive + * in sign call, but no matter how you twist it, something is bound to + * turn a little odd. + */ +/* + * Secret-key operations. + */ +void blst_keygen(blst_scalar *out_SK, const byte *IKM, size_t IKM_len, + const byte *info DEFNULL, size_t info_len DEFNULL); +void blst_sk_to_pk_in_g1(blst_p1 *out_pk, const blst_scalar *SK); +void blst_sign_pk_in_g1(blst_p2 *out_sig, const blst_p2 *hash, + const blst_scalar *SK); +void blst_sk_to_pk_in_g2(blst_p2 *out_pk, const blst_scalar *SK); +void blst_sign_pk_in_g2(blst_p1 *out_sig, const blst_p1 *hash, + const blst_scalar *SK); + +/* + * Pairing interface. + */ +#ifndef SWIG +void blst_miller_loop(blst_fp12 *ret, const blst_p2_affine *Q, + const blst_p1_affine *P); +void blst_final_exp(blst_fp12 *ret, const blst_fp12 *f); +void blst_precompute_lines(blst_fp6 Qlines[68], const blst_p2_affine *Q); +void blst_miller_loop_lines(blst_fp12 *ret, const blst_fp6 Qlines[68], + const blst_p1_affine *P); +bool blst_fp12_finalverify(const blst_fp12 *gt1, const blst_fp12 *gt2); +#endif + +#ifdef __BLST_CGO__ +typedef limb_t blst_pairing; +#elif defined(__BLST_RUST_BINDGEN__) +typedef struct {} blst_pairing; +#else +typedef struct blst_opaque blst_pairing; +#endif + +size_t blst_pairing_sizeof(); +void blst_pairing_init(blst_pairing *new_ctx, bool hash_or_encode, + const byte *DST DEFNULL, size_t DST_len DEFNULL); +const byte *blst_pairing_get_dst(const blst_pairing *ctx); +void blst_pairing_commit(blst_pairing *ctx); +BLST_ERROR blst_pairing_aggregate_pk_in_g2(blst_pairing *ctx, + const blst_p2_affine *PK, + const blst_p1_affine *signature, + const byte *msg, size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_chk_n_aggr_pk_in_g2(blst_pairing *ctx, + const blst_p2_affine *PK, + bool pk_grpchk, + const blst_p1_affine *signature, + bool sig_grpchk, + const byte *msg, size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_mul_n_aggregate_pk_in_g2(blst_pairing *ctx, + const blst_p2_affine *PK, + const blst_p1_affine *sig, + const byte *scalar, + size_t nbits, + const byte *msg, + size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_chk_n_mul_n_aggr_pk_in_g2(blst_pairing *ctx, + const blst_p2_affine *PK, + bool pk_grpchk, + const blst_p1_affine *sig, + bool sig_grpchk, + const byte *scalar, + size_t nbits, + const byte *msg, + size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_aggregate_pk_in_g1(blst_pairing *ctx, + const blst_p1_affine *PK, + const blst_p2_affine *signature, + const byte *msg, size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_chk_n_aggr_pk_in_g1(blst_pairing *ctx, + const blst_p1_affine *PK, + bool pk_grpchk, + const blst_p2_affine *signature, + bool sig_grpchk, + const byte *msg, size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_mul_n_aggregate_pk_in_g1(blst_pairing *ctx, + const blst_p1_affine *PK, + const blst_p2_affine *sig, + const byte *scalar, + size_t nbits, + const byte *msg, + size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_chk_n_mul_n_aggr_pk_in_g1(blst_pairing *ctx, + const blst_p1_affine *PK, + bool pk_grpchk, + const blst_p2_affine *sig, + bool sig_grpchk, + const byte *scalar, + size_t nbits, + const byte *msg, + size_t msg_len, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_pairing_merge(blst_pairing *ctx, const blst_pairing *ctx1); +bool blst_pairing_finalverify(const blst_pairing *ctx, + const blst_fp12 *gtsig DEFNULL); + + +/* + * Customarily applications aggregate signatures separately. + * In which case application would have to pass NULLs for |signature| + * to blst_pairing_aggregate calls and pass aggregated signature + * collected with these calls to blst_pairing_finalverify. Inputs are + * Zcash-compatible "straight-from-wire" byte vectors, compressed or + * not. + */ +BLST_ERROR blst_aggregate_in_g1(blst_p1 *out, const blst_p1 *in, + const byte *zwire); +BLST_ERROR blst_aggregate_in_g2(blst_p2 *out, const blst_p2 *in, + const byte *zwire); + +void blst_aggregated_in_g1(blst_fp12 *out, const blst_p1_affine *signature); +void blst_aggregated_in_g2(blst_fp12 *out, const blst_p2_affine *signature); + +/* + * "One-shot" CoreVerify entry points. + */ +BLST_ERROR blst_core_verify_pk_in_g1(const blst_p1_affine *pk, + const blst_p2_affine *signature, + bool hash_or_encode, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, + size_t DST_len DEFNULL, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); +BLST_ERROR blst_core_verify_pk_in_g2(const blst_p2_affine *pk, + const blst_p1_affine *signature, + bool hash_or_encode, + const byte *msg, size_t msg_len, + const byte *DST DEFNULL, + size_t DST_len DEFNULL, + const byte *aug DEFNULL, + size_t aug_len DEFNULL); + +extern const blst_p1_affine BLS12_381_G1; +extern const blst_p1_affine BLS12_381_NEG_G1; +extern const blst_p2_affine BLS12_381_G2; +extern const blst_p2_affine BLS12_381_NEG_G2; + +#include "blst_aux.h" + +#ifdef __cplusplus +} +#endif +#endif diff --git a/crypto/blst_src/blst_aux.h b/crypto/blst_src/blst_aux.h new file mode 100644 index 00000000000..6d444fc1729 --- /dev/null +++ b/crypto/blst_src/blst_aux.h @@ -0,0 +1,102 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLST_AUX_H__ +#define __BLST_AUX_H__ +/* + * This file lists interfaces that might be promoted to blst.h or removed, + * depending on their proven/unproven worthiness. + */ + +void blst_fr_to(blst_fr *ret, const blst_fr *a); +void blst_fr_from(blst_fr *ret, const blst_fr *a); + +void blst_fp_to(blst_fp *ret, const blst_fp *a); +void blst_fp_from(blst_fp *ret, const blst_fp *a); + +bool blst_fp_is_square(const blst_fp *a); +bool blst_fp2_is_square(const blst_fp2 *a); + +void blst_p1_from_jacobian(blst_p1 *out, const blst_p1 *in); +void blst_p2_from_jacobian(blst_p2 *out, const blst_p2 *in); + +/* + * Below functions produce both point and deserialized outcome of + * SkToPk and Sign. However, deserialized outputs are pre-decorated + * with sign and infinity bits. This means that you have to bring the + * output into compliance prior returning to application. If you want + * compressed point value, then do [equivalent of] + * + * byte temp[96]; + * blst_sk_to_pk2_in_g1(temp, out_pk, SK); + * temp[0] |= 0x80; + * memcpy(out, temp, 48); + * + * Otherwise do + * + * blst_sk_to_pk2_in_g1(out, out_pk, SK); + * out[0] &= ~0x20; + * + * Either |out| or |out_<point>| can be NULL. + */ +void blst_sk_to_pk2_in_g1(byte out[96], blst_p1_affine *out_pk, + const blst_scalar *SK); +void blst_sign_pk2_in_g1(byte out[192], blst_p2_affine *out_sig, + const blst_p2 *hash, const blst_scalar *SK); +void blst_sk_to_pk2_in_g2(byte out[192], blst_p2_affine *out_pk, + const blst_scalar *SK); +void blst_sign_pk2_in_g2(byte out[96], blst_p1_affine *out_sig, + const blst_p1 *hash, const blst_scalar *SK); + +typedef struct {} blst_uniq; + +size_t blst_uniq_sizeof(size_t n_nodes); +void blst_uniq_init(blst_uniq *tree); +bool blst_uniq_test(blst_uniq *tree, const byte *msg, size_t len); + +#ifdef expand_message_xmd +void expand_message_xmd(unsigned char *bytes, size_t len_in_bytes, + const unsigned char *aug, size_t aug_len, + const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len); +#else +void blst_expand_message_xmd(byte *out, size_t out_len, + const byte *msg, size_t msg_len, + const byte *DST, size_t DST_len); +#endif + +void blst_p1_unchecked_mult(blst_p1 *out, const blst_p1 *p, const byte *scalar, + size_t nbits); +void blst_p2_unchecked_mult(blst_p2 *out, const blst_p2 *p, const byte *scalar, + size_t nbits); + +void blst_pairing_raw_aggregate(blst_pairing *ctx, const blst_p2_affine *q, + const blst_p1_affine *p); +blst_fp12 *blst_pairing_as_fp12(blst_pairing *ctx); +void blst_bendian_from_fp12(byte out[48*12], const blst_fp12 *a); + +void blst_keygen_v3(blst_scalar *out_SK, const byte *IKM, size_t IKM_len, + const byte *info DEFNULL, size_t info_len DEFNULL); +void blst_keygen_v4_5(blst_scalar *out_SK, const byte *IKM, size_t IKM_len, + const byte *salt, size_t salt_len, + const byte *info DEFNULL, size_t info_len DEFNULL); +void blst_keygen_v5(blst_scalar *out_SK, const byte *IKM, size_t IKM_len, + const byte *salt, size_t salt_len, + const byte *info DEFNULL, size_t info_len DEFNULL); +void blst_derive_master_eip2333(blst_scalar *out_SK, + const byte *IKM, size_t IKM_len); +void blst_derive_child_eip2333(blst_scalar *out_SK, const blst_scalar *SK, + uint32_t child_index); + +void blst_scalar_from_hexascii(blst_scalar *out, const byte *hex); +void blst_fr_from_hexascii(blst_fr *ret, const byte *hex); +void blst_fp_from_hexascii(blst_fp *ret, const byte *hex); + +size_t blst_p1_sizeof(); +size_t blst_p1_affine_sizeof(); +size_t blst_p2_sizeof(); +size_t blst_p2_affine_sizeof(); +size_t blst_fp12_sizeof(); +#endif diff --git a/crypto/blst_src/build/bindings_trim.pl b/crypto/blst_src/build/bindings_trim.pl new file mode 100755 index 00000000000..90f914578d9 --- /dev/null +++ b/crypto/blst_src/build/bindings_trim.pl @@ -0,0 +1,37 @@ +#!/usr/bin/env perl + +# read whole file +while(<>) { push @file, $_; } + +# traverse and remove auto-generated PartialEq for chosen types +for (my $i = 0; $i <= $#file; $i++) { + if (@file[$i] =~ m/struct\s+blst_p[12]/) { + @file[$i-1] =~ s/,\s*PartialEq//; + } elsif (@file[$i] =~ m/struct\s+blst_fp12/) { + @file[$i-1] =~ s/,\s*(?:Default|PartialEq)//g; + } elsif (@file[$i] =~ m/struct\s+(blst_pairing|blst_uniq)/) { + @file[$i-1] =~ s/,\s*(?:Copy|Clone|Eq|PartialEq)//g; + } elsif (@file[$i] =~ m/struct\s+blst_scalar/) { + @file[$i-1] =~ s/,\s*Copy//; + @file[$i-1] =~ s/\)/, Zeroize\)/; + splice @file, $i, 0, "#[zeroize(drop)]\n"; $i++; + } elsif (@file[$i] =~ m/assert_eq!\($/) { + @file[++$i] =~ s/unsafe\s*\{\s*&\(\*\(::std::ptr::null::<(\w+)>\(\)\)\)\.(\w+).*\}/offsetof!($1, $2)/; + } +} + +print << '___'; +#[cfg(test)] +macro_rules! offsetof { + ($type:ty, $field:tt) => { + { + let v = <$type>::default(); + (&v.$field as *const _ as usize) - (&v as *const _ as usize) + } + }; +} +___ +# print the file +print @file; + +close STDOUT; diff --git a/crypto/blst_src/build/coff/add_mod_256-armv8.S b/crypto/blst_src/build/coff/add_mod_256-armv8.S new file mode 100644 index 00000000000..27b64ef4ca4 --- /dev/null +++ b/crypto/blst_src/build/coff/add_mod_256-armv8.S @@ -0,0 +1,397 @@ +.text + +.globl add_mod_256 + +.def add_mod_256; +.type 32; +.endef +.p2align 5 +add_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + adds x8,x8,x12 + ldp x14,x15,[x2,#16] + adcs x9,x9,x13 + ldp x4,x5,[x3] + adcs x10,x10,x14 + ldp x6,x7,[x3,#16] + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret + + +.globl mul_by_3_mod_256 + +.def mul_by_3_mod_256; +.type 32; +.endef +.p2align 5 +mul_by_3_mod_256: + ldp x12,x13,[x1] + ldp x14,x15,[x1,#16] + + adds x8,x12,x12 + ldp x4,x5,[x2] + adcs x9,x13,x13 + ldp x6,x7,[x2,#16] + adcs x10,x14,x14 + adcs x11,x15,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + adds x8,x8,x12 + adcs x9,x9,x13 + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret + + +.globl lshift_mod_256 + +.def lshift_mod_256; +.type 32; +.endef +.p2align 5 +lshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +.Loop_lshift_mod_256: + adds x8,x8,x8 + sub x2,x2,#1 + adcs x9,x9,x9 + adcs x10,x10,x10 + adcs x11,x11,x11 + adc x3,xzr,xzr + + subs x12,x8,x4 + sbcs x13,x9,x5 + sbcs x14,x10,x6 + sbcs x15,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x12,lo + csel x9,x9,x13,lo + csel x10,x10,x14,lo + csel x11,x11,x15,lo + + cbnz x2,.Loop_lshift_mod_256 + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + + +.globl rshift_mod_256 + +.def rshift_mod_256; +.type 32; +.endef +.p2align 5 +rshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +.Loop_rshift: + adds x12,x8,x4 + sub x2,x2,#1 + adcs x13,x9,x5 + adcs x14,x10,x6 + adcs x15,x11,x7 + adc x3,xzr,xzr + tst x8,#1 + + csel x12,x12,x8,ne + csel x13,x13,x9,ne + csel x14,x14,x10,ne + csel x15,x15,x11,ne + csel x3,x3,xzr,ne + + extr x8,x13,x12,#1 + extr x9,x14,x13,#1 + extr x10,x15,x14,#1 + extr x11,x3,x15,#1 + + cbnz x2,.Loop_rshift + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + + +.globl cneg_mod_256 + +.def cneg_mod_256; +.type 32; +.endef +.p2align 5 +cneg_mod_256: + ldp x8,x9,[x1] + ldp x4,x5,[x3] + + ldp x10,x11,[x1,#16] + subs x12,x4,x8 + ldp x6,x7,[x3,#16] + orr x4,x8,x9 + sbcs x13,x5,x9 + orr x5,x10,x11 + sbcs x14,x6,x10 + orr x3,x4,x5 + sbc x15,x7,x11 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x8,x8,x12,eq + csel x9,x9,x13,eq + csel x10,x10,x14,eq + stp x8,x9,[x0] + csel x11,x11,x15,eq + stp x10,x11,[x0,#16] + + ret + + +.globl sub_mod_256 + +.def sub_mod_256; +.type 32; +.endef +.p2align 5 +sub_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + subs x8,x8,x12 + ldp x14,x15,[x2,#16] + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + stp x8,x9,[x0] + adc x11,x11,x7 + stp x10,x11,[x0,#16] + + ret + + +.globl check_mod_256 + +.def check_mod_256; +.type 32; +.endef +.p2align 5 +check_mod_256: + ldp x8,x9,[x0] + ldp x10,x11,[x0,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + subs xzr,x8,x4 + sbcs xzr,x9,x5 + orr x8,x8,x9 + sbcs xzr,x10,x6 + orr x8,x8,x10 + sbcs xzr,x11,x7 + orr x8,x8,x11 + sbc x1,xzr,xzr + + cmp x8,#0 + mov x0,#1 + csel x0,x0,xzr,ne + and x0,x0,x1 + + ret + + +.globl add_n_check_mod_256 + +.def add_n_check_mod_256; +.type 32; +.endef +.p2align 5 +add_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + adds x8,x8,x12 + ldp x4,x5,[x3] + adcs x9,x9,x13 + ldp x6,x7,[x3,#16] + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret + + +.globl sub_n_check_mod_256 + +.def sub_n_check_mod_256; +.type 32; +.endef +.p2align 5 +sub_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + subs x8,x8,x12 + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + adc x11,x11,x7 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret + diff --git a/crypto/blst_src/build/coff/add_mod_256-x86_64.s b/crypto/blst_src/build/coff/add_mod_256-x86_64.s new file mode 100644 index 00000000000..f88e6189ca5 --- /dev/null +++ b/crypto/blst_src/build/coff/add_mod_256-x86_64.s @@ -0,0 +1,911 @@ +.text + +.globl add_mod_256 + +.def add_mod_256; .scl 2; .type 32; .endef +.p2align 5 +add_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_add_mod_256: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loaded_a_add_mod_256: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_add_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_add_mod_256: + + +.globl mul_by_3_mod_256 + +.def mul_by_3_mod_256; .scl 2; .type 32; .endef +.p2align 5 +mul_by_3_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_3_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + +.LSEH_body_mul_by_3_mod_256: + + + movq %rdx,%rcx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rsi,%rdx + movq 24(%rsi),%r11 + + call __lshift_mod_256 + movq 0(%rsp),%r12 + + jmp .Loaded_a_add_mod_256 + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_mul_by_3_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_3_mod_256: + +.def __lshift_mod_256; .scl 3; .type 32; .endef +.p2align 5 +__lshift_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + movq %r8,%rax + adcq %r10,%r10 + movq %r9,%rsi + adcq %r11,%r11 + sbbq %r12,%r12 + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + cmovcq %rbx,%r10 + cmovcq %rbp,%r11 + + .byte 0xf3,0xc3 + + + +.globl lshift_mod_256 + +.def lshift_mod_256; .scl 2; .type 32; .endef +.p2align 5 +lshift_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_lshift_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + +.LSEH_body_lshift_mod_256: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loop_lshift_mod_256: + call __lshift_mod_256 + decl %edx + jnz .Loop_lshift_mod_256 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 0(%rsp),%r12 + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_lshift_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_lshift_mod_256: + + +.globl rshift_mod_256 + +.def rshift_mod_256; .scl 2; .type 32; .endef +.p2align 5 +rshift_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_rshift_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_rshift_mod_256: + + + movq 0(%rsi),%rbp + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loop_rshift_mod_256: + movq %rbp,%r8 + andq $1,%rbp + movq 0(%rcx),%rax + negq %rbp + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + + andq %rbp,%rax + andq %rbp,%rsi + andq %rbp,%rbx + andq 24(%rcx),%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + adcq %rbx,%r10 + adcq %rbp,%r11 + sbbq %rax,%rax + + shrq $1,%r8 + movq %r9,%rbp + shrq $1,%r9 + movq %r10,%rbx + shrq $1,%r10 + movq %r11,%rsi + shrq $1,%r11 + + shlq $63,%rbp + shlq $63,%rbx + orq %r8,%rbp + shlq $63,%rsi + orq %rbx,%r9 + shlq $63,%rax + orq %rsi,%r10 + orq %rax,%r11 + + decl %edx + jnz .Loop_rshift_mod_256 + + movq %rbp,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_rshift_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_rshift_mod_256: + + +.globl cneg_mod_256 + +.def cneg_mod_256; .scl 2; .type 32; .endef +.p2align 5 +cneg_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_cneg_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + +.LSEH_body_cneg_mod_256: + + + movq 0(%rsi),%r12 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %r12,%r8 + movq 24(%rsi),%r11 + orq %r9,%r12 + orq %r10,%r12 + orq %r11,%r12 + movq $-1,%rbp + + movq 0(%rcx),%rax + cmovnzq %rbp,%r12 + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + andq %r12,%rax + movq 24(%rcx),%rbp + andq %r12,%rsi + andq %r12,%rbx + andq %r12,%rbp + + subq %r8,%rax + sbbq %r9,%rsi + sbbq %r10,%rbx + sbbq %r11,%rbp + + orq %rdx,%rdx + + cmovzq %r8,%rax + cmovzq %r9,%rsi + movq %rax,0(%rdi) + cmovzq %r10,%rbx + movq %rsi,8(%rdi) + cmovzq %r11,%rbp + movq %rbx,16(%rdi) + movq %rbp,24(%rdi) + + movq 0(%rsp),%r12 + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_cneg_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_cneg_mod_256: + + +.globl sub_mod_256 + +.def sub_mod_256; .scl 2; .type 32; .endef +.p2align 5 +sub_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_sub_mod_256: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_sub_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sub_mod_256: + + +.globl check_mod_256 + +.def check_mod_256; .scl 2; .type 32; .endef +.p2align 5 +check_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_check_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + + + movq 0(%rdi),%rax + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + + movq %rax,%r8 + orq %r9,%rax + orq %r10,%rax + orq %r11,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq %rsi,%rsi + + movq $1,%rdx + cmpq $0,%rax + cmovneq %rdx,%rax + andq %rsi,%rax +.LSEH_epilogue_check_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_check_mod_256: + + +.globl add_n_check_mod_256 + +.def add_n_check_mod_256; .scl 2; .type 32; .endef +.p2align 5 +add_n_check_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_n_check_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_add_n_check_mod_256: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_add_n_check_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_add_n_check_mod_256: + + +.globl sub_n_check_mod_256 + +.def sub_n_check_mod_256; .scl 2; .type 32; .endef +.p2align 5 +sub_n_check_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_n_check_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_sub_n_check_mod_256: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_sub_n_check_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sub_n_check_mod_256: +.section .pdata +.p2align 2 +.rva .LSEH_begin_add_mod_256 +.rva .LSEH_body_add_mod_256 +.rva .LSEH_info_add_mod_256_prologue + +.rva .LSEH_body_add_mod_256 +.rva .LSEH_epilogue_add_mod_256 +.rva .LSEH_info_add_mod_256_body + +.rva .LSEH_epilogue_add_mod_256 +.rva .LSEH_end_add_mod_256 +.rva .LSEH_info_add_mod_256_epilogue + +.rva .LSEH_begin_mul_by_3_mod_256 +.rva .LSEH_body_mul_by_3_mod_256 +.rva .LSEH_info_mul_by_3_mod_256_prologue + +.rva .LSEH_body_mul_by_3_mod_256 +.rva .LSEH_epilogue_mul_by_3_mod_256 +.rva .LSEH_info_mul_by_3_mod_256_body + +.rva .LSEH_epilogue_mul_by_3_mod_256 +.rva .LSEH_end_mul_by_3_mod_256 +.rva .LSEH_info_mul_by_3_mod_256_epilogue + +.rva .LSEH_begin_lshift_mod_256 +.rva .LSEH_body_lshift_mod_256 +.rva .LSEH_info_lshift_mod_256_prologue + +.rva .LSEH_body_lshift_mod_256 +.rva .LSEH_epilogue_lshift_mod_256 +.rva .LSEH_info_lshift_mod_256_body + +.rva .LSEH_epilogue_lshift_mod_256 +.rva .LSEH_end_lshift_mod_256 +.rva .LSEH_info_lshift_mod_256_epilogue + +.rva .LSEH_begin_rshift_mod_256 +.rva .LSEH_body_rshift_mod_256 +.rva .LSEH_info_rshift_mod_256_prologue + +.rva .LSEH_body_rshift_mod_256 +.rva .LSEH_epilogue_rshift_mod_256 +.rva .LSEH_info_rshift_mod_256_body + +.rva .LSEH_epilogue_rshift_mod_256 +.rva .LSEH_end_rshift_mod_256 +.rva .LSEH_info_rshift_mod_256_epilogue + +.rva .LSEH_begin_cneg_mod_256 +.rva .LSEH_body_cneg_mod_256 +.rva .LSEH_info_cneg_mod_256_prologue + +.rva .LSEH_body_cneg_mod_256 +.rva .LSEH_epilogue_cneg_mod_256 +.rva .LSEH_info_cneg_mod_256_body + +.rva .LSEH_epilogue_cneg_mod_256 +.rva .LSEH_end_cneg_mod_256 +.rva .LSEH_info_cneg_mod_256_epilogue + +.rva .LSEH_begin_sub_mod_256 +.rva .LSEH_body_sub_mod_256 +.rva .LSEH_info_sub_mod_256_prologue + +.rva .LSEH_body_sub_mod_256 +.rva .LSEH_epilogue_sub_mod_256 +.rva .LSEH_info_sub_mod_256_body + +.rva .LSEH_epilogue_sub_mod_256 +.rva .LSEH_end_sub_mod_256 +.rva .LSEH_info_sub_mod_256_epilogue + +.rva .LSEH_epilogue_check_mod_256 +.rva .LSEH_end_check_mod_256 +.rva .LSEH_info_check_mod_256_epilogue + +.rva .LSEH_begin_add_n_check_mod_256 +.rva .LSEH_body_add_n_check_mod_256 +.rva .LSEH_info_add_n_check_mod_256_prologue + +.rva .LSEH_body_add_n_check_mod_256 +.rva .LSEH_epilogue_add_n_check_mod_256 +.rva .LSEH_info_add_n_check_mod_256_body + +.rva .LSEH_epilogue_add_n_check_mod_256 +.rva .LSEH_end_add_n_check_mod_256 +.rva .LSEH_info_add_n_check_mod_256_epilogue + +.rva .LSEH_begin_sub_n_check_mod_256 +.rva .LSEH_body_sub_n_check_mod_256 +.rva .LSEH_info_sub_n_check_mod_256_prologue + +.rva .LSEH_body_sub_n_check_mod_256 +.rva .LSEH_epilogue_sub_n_check_mod_256 +.rva .LSEH_info_sub_n_check_mod_256_body + +.rva .LSEH_epilogue_sub_n_check_mod_256 +.rva .LSEH_end_sub_n_check_mod_256 +.rva .LSEH_info_sub_n_check_mod_256_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_add_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_add_mod_256_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_add_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_3_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_3_mod_256_body: +.byte 1,0,11,0 +.byte 0x00,0xc4,0x00,0x00 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00,0x00,0x00,0x00,0x00 +.LSEH_info_mul_by_3_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_lshift_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_lshift_mod_256_body: +.byte 1,0,11,0 +.byte 0x00,0xc4,0x00,0x00 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00,0x00,0x00,0x00,0x00 +.LSEH_info_lshift_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_rshift_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_rshift_mod_256_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_rshift_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_cneg_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_cneg_mod_256_body: +.byte 1,0,11,0 +.byte 0x00,0xc4,0x00,0x00 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00,0x00,0x00,0x00,0x00 +.LSEH_info_cneg_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sub_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sub_mod_256_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_sub_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_check_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_add_n_check_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_add_n_check_mod_256_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_add_n_check_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sub_n_check_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sub_n_check_mod_256_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_sub_n_check_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/add_mod_384-armv8.S b/crypto/blst_src/build/coff/add_mod_384-armv8.S new file mode 100644 index 00000000000..2eff0677f54 --- /dev/null +++ b/crypto/blst_src/build/coff/add_mod_384-armv8.S @@ -0,0 +1,1056 @@ +.text + +.globl add_mod_384 + +.def add_mod_384; +.type 32; +.endef +.p2align 5 +add_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.def __add_mod_384; +.type 32; +.endef +.p2align 5 +__add_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + +__add_mod_384_ab_are_loaded: + adds x10,x10,x16 + adcs x11,x11,x17 + adcs x12,x12,x19 + adcs x13,x13,x20 + adcs x14,x14,x21 + adcs x15,x15,x22 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret + + +.globl add_mod_384x + +.def add_mod_384x; +.type 32; +.endef +.p2align 5 +add_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl rshift_mod_384 + +.def rshift_mod_384; +.type 32; +.endef +.p2align 5 +rshift_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +.Loop_rshift_mod_384: + sub x2,x2,#1 + bl __rshift_mod_384 + cbnz x2,.Loop_rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.def __rshift_mod_384; +.type 32; +.endef +.p2align 5 +__rshift_mod_384: + sbfx x22,x10,#0,#1 + and x16,x22,x4 + and x17,x22,x5 + adds x10,x10,x16 + and x19,x22,x6 + adcs x11,x11,x17 + and x20,x22,x7 + adcs x12,x12,x19 + and x21,x22,x8 + adcs x13,x13,x20 + and x22,x22,x9 + adcs x14,x14,x21 + extr x10,x11,x10,#1 // a[0:5] >>= 1 + adcs x15,x15,x22 + extr x11,x12,x11,#1 + adc x22,xzr,xzr + extr x12,x13,x12,#1 + extr x13,x14,x13,#1 + extr x14,x15,x14,#1 + extr x15,x22,x15,#1 + ret + + +.globl div_by_2_mod_384 + +.def div_by_2_mod_384; +.type 32; +.endef +.p2align 5 +div_by_2_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl lshift_mod_384 + +.def lshift_mod_384; +.type 32; +.endef +.p2align 5 +lshift_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +.Loop_lshift_mod_384: + sub x2,x2,#1 + bl __lshift_mod_384 + cbnz x2,.Loop_lshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.def __lshift_mod_384; +.type 32; +.endef +.p2align 5 +__lshift_mod_384: + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret + + +.globl mul_by_3_mod_384 + +.def mul_by_3_mod_384; +.type 32; +.endef +.p2align 5 +mul_by_3_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl mul_by_8_mod_384 + +.def mul_by_8_mod_384; +.type 32; +.endef +.p2align 5 +mul_by_8_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl mul_by_3_mod_384x + +.def mul_by_3_mod_384x; +.type 32; +.endef +.p2align 5 +mul_by_3_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + + ldp x16,x17,[x1,#48] + ldp x19,x20,[x1,#64] + ldp x21,x22,[x1,#80] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl mul_by_8_mod_384x + +.def mul_by_8_mod_384x; +.type 32; +.endef +.p2align 5 +mul_by_8_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl cneg_mod_384 + +.def cneg_mod_384; +.type 32; +.endef +.p2align 5 +cneg_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x4,x5,[x3] + ldp x12,x13,[x1,#16] + ldp x6,x7,[x3,#16] + + subs x16,x4,x10 + ldp x14,x15,[x1,#32] + ldp x8,x9,[x3,#32] + orr x3,x10,x11 + sbcs x17,x5,x11 + orr x3,x3,x12 + sbcs x19,x6,x12 + orr x3,x3,x13 + sbcs x20,x7,x13 + orr x3,x3,x14 + sbcs x21,x8,x14 + orr x3,x3,x15 + sbc x22,x9,x15 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x10,x10,x16,eq + csel x11,x11,x17,eq + csel x12,x12,x19,eq + csel x13,x13,x20,eq + stp x10,x11,[x0] + csel x14,x14,x21,eq + stp x12,x13,[x0,#16] + csel x15,x15,x22,eq + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl sub_mod_384 + +.def sub_mod_384; +.type 32; +.endef +.p2align 5 +sub_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.def __sub_mod_384; +.type 32; +.endef +.p2align 5 +__sub_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + + subs x10,x10,x16 + sbcs x11,x11,x17 + sbcs x12,x12,x19 + sbcs x13,x13,x20 + sbcs x14,x14,x21 + sbcs x15,x15,x22 + sbc x3,xzr,xzr + + and x16,x4,x3 + and x17,x5,x3 + adds x10,x10,x16 + and x19,x6,x3 + adcs x11,x11,x17 + and x20,x7,x3 + adcs x12,x12,x19 + and x21,x8,x3 + adcs x13,x13,x20 + and x22,x9,x3 + adcs x14,x14,x21 + adc x15,x15,x22 + + ret + + +.globl sub_mod_384x + +.def sub_mod_384x; +.type 32; +.endef +.p2align 5 +sub_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl mul_by_1_plus_i_mod_384x + +.def mul_by_1_plus_i_mod_384x; +.type 32; +.endef +.p2align 5 +mul_by_1_plus_i_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + add x2,x1,#48 + + bl __sub_mod_384 // a->re - a->im + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __add_mod_384_ab_are_loaded // a->re + a->im + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl sgn0_pty_mod_384 + +.def sgn0_pty_mod_384; +.type 32; +.endef +.p2align 5 +sgn0_pty_mod_384: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x0,x10,#1 + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x3,x3,xzr + + mvn x3,x3 + and x3,x3,#2 + orr x0,x0,x3 + + ret + + +.globl sgn0_pty_mod_384x + +.def sgn0_pty_mod_384x; +.type 32; +.endef +.p2align 5 +sgn0_pty_mod_384x: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x2,x10,#1 + orr x3,x10,x11 + adds x10,x10,x10 + orr x3,x3,x12 + adcs x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + ldp x10,x11,[x0,#48] + ldp x12,x13,[x0,#64] + ldp x14,x15,[x0,#80] + + mvn x16,x16 + and x16,x16,#2 + orr x2,x2,x16 + + and x0,x10,#1 + orr x1,x10,x11 + adds x10,x10,x10 + orr x1,x1,x12 + adcs x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + mvn x16,x16 + and x16,x16,#2 + orr x0,x0,x16 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ret + +.globl vec_select_32 + +.def vec_select_32; +.type 32; +.endef +.p2align 5 +vec_select_32: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl vec_select_48 + +.def vec_select_48; +.type 32; +.endef +.p2align 5 +vec_select_48: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl vec_select_96 + +.def vec_select_96; +.type 32; +.endef +.p2align 5 +vec_select_96: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl vec_select_192 + +.def vec_select_192; +.type 32; +.endef +.p2align 5 +vec_select_192: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl vec_select_144 + +.def vec_select_144; +.type 32; +.endef +.p2align 5 +vec_select_144: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl vec_select_288 + +.def vec_select_288; +.type 32; +.endef +.p2align 5 +vec_select_288: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl vec_prefetch + +.def vec_prefetch; +.type 32; +.endef +.p2align 5 +vec_prefetch: + add x1, x1, x0 + sub x1, x1, #1 + mov x2, #64 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + prfm pldl1keep, [x0] + ret + +.globl vec_is_zero_16x + +.def vec_is_zero_16x; +.type 32; +.endef +.p2align 5 +vec_is_zero_16x: + ld1 {v0.2d}, [x0], #16 + lsr x1, x1, #4 + sub x1, x1, #1 + cbz x1, .Loop_is_zero_done + +.Loop_is_zero: + ld1 {v1.2d}, [x0], #16 + orr v0.16b, v0.16b, v1.16b + sub x1, x1, #1 + cbnz x1, .Loop_is_zero + +.Loop_is_zero_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret + +.globl vec_is_equal_16x + +.def vec_is_equal_16x; +.type 32; +.endef +.p2align 5 +vec_is_equal_16x: + ld1 {v0.2d}, [x0], #16 + ld1 {v1.2d}, [x1], #16 + lsr x2, x2, #4 + eor v0.16b, v0.16b, v1.16b + +.Loop_is_equal: + sub x2, x2, #1 + cbz x2, .Loop_is_equal_done + ld1 {v1.2d}, [x0], #16 + ld1 {v2.2d}, [x1], #16 + eor v1.16b, v1.16b, v2.16b + orr v0.16b, v0.16b, v1.16b + b .Loop_is_equal + nop + +.Loop_is_equal_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret + diff --git a/crypto/blst_src/build/coff/add_mod_384-x86_64.s b/crypto/blst_src/build/coff/add_mod_384-x86_64.s new file mode 100644 index 00000000000..d1c7ad6e689 --- /dev/null +++ b/crypto/blst_src/build/coff/add_mod_384-x86_64.s @@ -0,0 +1,2481 @@ +.text + +.globl add_mod_384 + +.def add_mod_384; .scl 2; .type 32; .endef +.p2align 5 +add_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_add_mod_384: + + + call __add_mod_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_add_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_add_mod_384: + +.def __add_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__add_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__add_mod_384_a_is_loaded: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + + +.globl add_mod_384x + +.def add_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +add_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $24,%rsp + +.LSEH_body_add_mod_384x: + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __add_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + movq 24+0(%rsp),%r15 + + movq 24+8(%rsp),%r14 + + movq 24+16(%rsp),%r13 + + movq 24+24(%rsp),%r12 + + movq 24+32(%rsp),%rbx + + movq 24+40(%rsp),%rbp + + leaq 24+48(%rsp),%rsp + +.LSEH_epilogue_add_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_add_mod_384x: + + +.globl rshift_mod_384 + +.def rshift_mod_384; .scl 2; .type 32; .endef +.p2align 5 +rshift_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_rshift_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_rshift_mod_384: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +.Loop_rshift_mod_384: + call __rshift_mod_384 + decl %edx + jnz .Loop_rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_rshift_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_rshift_mod_384: + +.def __rshift_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__rshift_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rsi + movq 0(%rcx),%r14 + andq %r8,%rsi + movq 8(%rcx),%r15 + negq %rsi + movq 16(%rcx),%rax + andq %rsi,%r14 + movq 24(%rcx),%rbx + andq %rsi,%r15 + movq 32(%rcx),%rbp + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%rbx + adcq %r12,%rbp + adcq %r13,%rsi + sbbq %r13,%r13 + + shrq $1,%r14 + movq %r15,%r8 + shrq $1,%r15 + movq %rax,%r9 + shrq $1,%rax + movq %rbx,%r10 + shrq $1,%rbx + movq %rbp,%r11 + shrq $1,%rbp + movq %rsi,%r12 + shrq $1,%rsi + shlq $63,%r8 + shlq $63,%r9 + orq %r14,%r8 + shlq $63,%r10 + orq %r15,%r9 + shlq $63,%r11 + orq %rax,%r10 + shlq $63,%r12 + orq %rbx,%r11 + shlq $63,%r13 + orq %rbp,%r12 + orq %rsi,%r13 + + .byte 0xf3,0xc3 + + +.globl div_by_2_mod_384 + +.def div_by_2_mod_384; .scl 2; .type 32; .endef +.p2align 5 +div_by_2_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_div_by_2_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_div_by_2_mod_384: + + + movq 0(%rsi),%r8 + movq %rdx,%rcx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + call __rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_div_by_2_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_div_by_2_mod_384: + + +.globl lshift_mod_384 + +.def lshift_mod_384; .scl 2; .type 32; .endef +.p2align 5 +lshift_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_lshift_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_lshift_mod_384: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +.Loop_lshift_mod_384: + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdi,%rdi + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdi + + movq (%rsp),%rdi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + decl %edx + jnz .Loop_lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_lshift_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_lshift_mod_384: + +.def __lshift_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__lshift_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + .byte 0xf3,0xc3 + + + +.globl mul_by_3_mod_384 + +.def mul_by_3_mod_384; .scl 2; .type 32; .endef +.p2align 5 +mul_by_3_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_3_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rsi + +.LSEH_body_mul_by_3_mod_384: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mul_by_3_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_3_mod_384: + +.globl mul_by_8_mod_384 + +.def mul_by_8_mod_384; .scl 2; .type 32; .endef +.p2align 5 +mul_by_8_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_8_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_mul_by_8_mod_384: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mul_by_8_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_8_mod_384: + + +.globl mul_by_3_mod_384x + +.def mul_by_3_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +mul_by_3_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_3_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rsi + +.LSEH_body_mul_by_3_mod_384x: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq (%rsp),%rsi + leaq 48(%rdi),%rdi + + movq 48(%rsi),%r8 + movq 56(%rsi),%r9 + movq 64(%rsi),%r10 + movq 72(%rsi),%r11 + movq 80(%rsi),%r12 + movq 88(%rsi),%r13 + + call __lshift_mod_384 + + movq $48,%rdx + addq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mul_by_3_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_3_mod_384x: + +.globl mul_by_8_mod_384x + +.def mul_by_8_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +mul_by_8_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_8_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rsi + +.LSEH_body_mul_by_8_mod_384x: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq (%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,48+0(%rdi) + movq %r9,48+8(%rdi) + movq %r10,48+16(%rdi) + movq %r11,48+24(%rdi) + movq %r12,48+32(%rdi) + movq %r13,48+40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mul_by_8_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_8_mod_384x: + + +.globl cneg_mod_384 + +.def cneg_mod_384; .scl 2; .type 32; .endef +.p2align 5 +cneg_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_cneg_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdx + +.LSEH_body_cneg_mod_384: + + + movq 0(%rsi),%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rdx,%r8 + movq 24(%rsi),%r11 + orq %r9,%rdx + movq 32(%rsi),%r12 + orq %r10,%rdx + movq 40(%rsi),%r13 + orq %r11,%rdx + movq $-1,%rsi + orq %r12,%rdx + orq %r13,%rdx + + movq 0(%rcx),%r14 + cmovnzq %rsi,%rdx + movq 8(%rcx),%r15 + movq 16(%rcx),%rax + andq %rdx,%r14 + movq 24(%rcx),%rbx + andq %rdx,%r15 + movq 32(%rcx),%rbp + andq %rdx,%rax + movq 40(%rcx),%rsi + andq %rdx,%rbx + movq 0(%rsp),%rcx + andq %rdx,%rbp + andq %rdx,%rsi + + subq %r8,%r14 + sbbq %r9,%r15 + sbbq %r10,%rax + sbbq %r11,%rbx + sbbq %r12,%rbp + sbbq %r13,%rsi + + orq %rcx,%rcx + + cmovzq %r8,%r14 + cmovzq %r9,%r15 + cmovzq %r10,%rax + movq %r14,0(%rdi) + cmovzq %r11,%rbx + movq %r15,8(%rdi) + cmovzq %r12,%rbp + movq %rax,16(%rdi) + cmovzq %r13,%rsi + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rsi,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_cneg_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_cneg_mod_384: + + +.globl sub_mod_384 + +.def sub_mod_384; .scl 2; .type 32; .endef +.p2align 5 +sub_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sub_mod_384: + + + call __sub_mod_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sub_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sub_mod_384: + +.def __sub_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + + +.globl sub_mod_384x + +.def sub_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +sub_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $24,%rsp + +.LSEH_body_sub_mod_384x: + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __sub_mod_384 + + movq 24+0(%rsp),%r15 + + movq 24+8(%rsp),%r14 + + movq 24+16(%rsp),%r13 + + movq 24+24(%rsp),%r12 + + movq 24+32(%rsp),%rbx + + movq 24+40(%rsp),%rbp + + leaq 24+48(%rsp),%rsp + +.LSEH_epilogue_sub_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sub_mod_384x: +.globl mul_by_1_plus_i_mod_384x + +.def mul_by_1_plus_i_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +mul_by_1_plus_i_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_by_1_plus_i_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $56,%rsp + +.LSEH_body_mul_by_1_plus_i_mod_384x: + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rbx + adcq 72(%rsi),%r11 + movq %r12,%rcx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + movq %rdi,48(%rsp) + sbbq %rdi,%rdi + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rbx + sbbq 80(%rsi),%rcx + sbbq 88(%rsi),%rbp + sbbq %rsi,%rsi + + movq %r8,0(%rsp) + movq 0(%rdx),%r8 + movq %r9,8(%rsp) + movq 8(%rdx),%r9 + movq %r10,16(%rsp) + movq 16(%rdx),%r10 + movq %r11,24(%rsp) + movq 24(%rdx),%r11 + movq %r12,32(%rsp) + andq %rsi,%r8 + movq 32(%rdx),%r12 + movq %r13,40(%rsp) + andq %rsi,%r9 + movq 40(%rdx),%r13 + andq %rsi,%r10 + andq %rsi,%r11 + andq %rsi,%r12 + andq %rsi,%r13 + movq 48(%rsp),%rsi + + addq %r8,%r14 + movq 0(%rsp),%r8 + adcq %r9,%r15 + movq 8(%rsp),%r9 + adcq %r10,%rax + movq 16(%rsp),%r10 + adcq %r11,%rbx + movq 24(%rsp),%r11 + adcq %r12,%rcx + movq 32(%rsp),%r12 + adcq %r13,%rbp + movq 40(%rsp),%r13 + + movq %r14,0(%rsi) + movq %r8,%r14 + movq %r15,8(%rsi) + movq %rax,16(%rsi) + movq %r9,%r15 + movq %rbx,24(%rsi) + movq %rcx,32(%rsi) + movq %r10,%rax + movq %rbp,40(%rsi) + + subq 0(%rdx),%r8 + movq %r11,%rbx + sbbq 8(%rdx),%r9 + sbbq 16(%rdx),%r10 + movq %r12,%rcx + sbbq 24(%rdx),%r11 + sbbq 32(%rdx),%r12 + movq %r13,%rbp + sbbq 40(%rdx),%r13 + sbbq $0,%rdi + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,48(%rsi) + cmovcq %rbx,%r11 + movq %r9,56(%rsi) + cmovcq %rcx,%r12 + movq %r10,64(%rsi) + cmovcq %rbp,%r13 + movq %r11,72(%rsi) + movq %r12,80(%rsi) + movq %r13,88(%rsi) + + movq 56+0(%rsp),%r15 + + movq 56+8(%rsp),%r14 + + movq 56+16(%rsp),%r13 + + movq 56+24(%rsp),%r12 + + movq 56+32(%rsp),%rbx + + movq 56+40(%rsp),%rbp + + leaq 56+48(%rsp),%rsp + +.LSEH_epilogue_mul_by_1_plus_i_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_by_1_plus_i_mod_384x: +.globl sgn0_pty_mod_384 + +.def sgn0_pty_mod_384; .scl 2; .type 32; .endef +.p2align 5 +sgn0_pty_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0_pty_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + + +.LSEH_body_sgn0_pty_mod_384: + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%rcx + movq 40(%rdi),%rdx + + xorq %rax,%rax + movq %r8,%rdi + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + notq %rax + andq $1,%rdi + andq $2,%rax + orq %rdi,%rax + +.LSEH_epilogue_sgn0_pty_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0_pty_mod_384: + +.globl sgn0_pty_mod_384x + +.def sgn0_pty_mod_384x; .scl 2; .type 32; .endef +.p2align 5 +sgn0_pty_mod_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0_pty_mod_384x: + movq %rcx,%rdi + movq %rdx,%rsi + + + pushq %rbp + + pushq %rbx + + subq $8,%rsp + +.LSEH_body_sgn0_pty_mod_384x: + + + movq 48(%rdi),%r8 + movq 56(%rdi),%r9 + movq 64(%rdi),%r10 + movq 72(%rdi),%r11 + movq 80(%rdi),%rcx + movq 88(%rdi),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + leaq 0(%rdi),%rax + xorq %rdi,%rdi + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rdi + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rdi + + movq %r8,0(%rsp) + notq %rdi + andq $1,%rbp + andq $2,%rdi + orq %rbp,%rdi + + movq 0(%rax),%r8 + movq 8(%rax),%r9 + movq 16(%rax),%r10 + movq 24(%rax),%r11 + movq 32(%rax),%rcx + movq 40(%rax),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + xorq %rax,%rax + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + movq 0(%rsp),%rbx + + notq %rax + + testq %r8,%r8 + cmovzq %rdi,%rbp + + testq %rbx,%rbx + cmovnzq %rdi,%rax + + andq $1,%rbp + andq $2,%rax + orq %rbp,%rax + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_sgn0_pty_mod_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0_pty_mod_384x: +.globl vec_select_32 + +.def vec_select_32; .scl 2; .type 32; .endef +.p2align 5 +vec_select_32: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 16(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 16(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 16(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-16(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-16(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-16(%rcx) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,16-16(%rcx) + .byte 0xf3,0xc3 + +.globl vec_select_48 + +.def vec_select_48; .scl 2; .type 32; .endef +.p2align 5 +vec_select_48: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 24(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 24(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 24(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-24(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-24(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-24(%rcx) + pand %xmm4,%xmm2 + movdqu 16+16-24(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-24(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-24(%rcx) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,32-24(%rcx) + .byte 0xf3,0xc3 + +.globl vec_select_96 + +.def vec_select_96; .scl 2; .type 32; .endef +.p2align 5 +vec_select_96: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 48(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 48(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 48(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-48(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-48(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-48(%rcx) + pand %xmm4,%xmm2 + movdqu 16+16-48(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-48(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-48(%rcx) + pand %xmm4,%xmm0 + movdqu 32+16-48(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-48(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-48(%rcx) + pand %xmm4,%xmm2 + movdqu 48+16-48(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-48(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-48(%rcx) + pand %xmm4,%xmm0 + movdqu 64+16-48(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-48(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-48(%rcx) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,80-48(%rcx) + .byte 0xf3,0xc3 + +.globl vec_select_192 + +.def vec_select_192; .scl 2; .type 32; .endef +.p2align 5 +vec_select_192: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 96(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 96(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 96(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-96(%rcx) + pand %xmm4,%xmm2 + movdqu 16+16-96(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-96(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-96(%rcx) + pand %xmm4,%xmm0 + movdqu 32+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-96(%rcx) + pand %xmm4,%xmm2 + movdqu 48+16-96(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-96(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-96(%rcx) + pand %xmm4,%xmm0 + movdqu 64+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-96(%rcx) + pand %xmm4,%xmm2 + movdqu 80+16-96(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-96(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-96(%rcx) + pand %xmm4,%xmm0 + movdqu 96+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-96(%rcx) + pand %xmm4,%xmm2 + movdqu 112+16-96(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-96(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-96(%rcx) + pand %xmm4,%xmm0 + movdqu 128+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-96(%rcx) + pand %xmm4,%xmm2 + movdqu 144+16-96(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-96(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-96(%rcx) + pand %xmm4,%xmm0 + movdqu 160+16-96(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-96(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-96(%rcx) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,176-96(%rcx) + .byte 0xf3,0xc3 + +.globl vec_select_144 + +.def vec_select_144; .scl 2; .type 32; .endef +.p2align 5 +vec_select_144: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 72(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 72(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 72(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-72(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-72(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-72(%rcx) + pand %xmm4,%xmm2 + movdqu 16+16-72(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-72(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-72(%rcx) + pand %xmm4,%xmm0 + movdqu 32+16-72(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-72(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-72(%rcx) + pand %xmm4,%xmm2 + movdqu 48+16-72(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-72(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-72(%rcx) + pand %xmm4,%xmm0 + movdqu 64+16-72(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-72(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-72(%rcx) + pand %xmm4,%xmm2 + movdqu 80+16-72(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-72(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-72(%rcx) + pand %xmm4,%xmm0 + movdqu 96+16-72(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-72(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-72(%rcx) + pand %xmm4,%xmm2 + movdqu 112+16-72(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-72(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-72(%rcx) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,128-72(%rcx) + .byte 0xf3,0xc3 + +.globl vec_select_288 + +.def vec_select_288; .scl 2; .type 32; .endef +.p2align 5 +vec_select_288: + .byte 0xf3,0x0f,0x1e,0xfa + + movd %r9d,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rdx),%xmm0 + leaq 144(%rdx),%rdx + pcmpeqd %xmm4,%xmm5 + movdqu (%r8),%xmm1 + leaq 144(%r8),%r8 + pcmpeqd %xmm5,%xmm4 + leaq 144(%rcx),%rcx + pand %xmm4,%xmm0 + movdqu 0+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-144(%rcx) + pand %xmm4,%xmm2 + movdqu 16+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-144(%rcx) + pand %xmm4,%xmm0 + movdqu 32+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-144(%rcx) + pand %xmm4,%xmm2 + movdqu 48+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-144(%rcx) + pand %xmm4,%xmm0 + movdqu 64+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-144(%rcx) + pand %xmm4,%xmm2 + movdqu 80+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-144(%rcx) + pand %xmm4,%xmm0 + movdqu 96+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-144(%rcx) + pand %xmm4,%xmm2 + movdqu 112+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-144(%rcx) + pand %xmm4,%xmm0 + movdqu 128+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-144(%rcx) + pand %xmm4,%xmm2 + movdqu 144+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-144(%rcx) + pand %xmm4,%xmm0 + movdqu 160+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-144(%rcx) + pand %xmm4,%xmm2 + movdqu 176+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 176+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,176-144(%rcx) + pand %xmm4,%xmm0 + movdqu 192+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 192+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,192-144(%rcx) + pand %xmm4,%xmm2 + movdqu 208+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 208+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,208-144(%rcx) + pand %xmm4,%xmm0 + movdqu 224+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 224+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,224-144(%rcx) + pand %xmm4,%xmm2 + movdqu 240+16-144(%rdx),%xmm0 + pand %xmm5,%xmm3 + movdqu 240+16-144(%r8),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,240-144(%rcx) + pand %xmm4,%xmm0 + movdqu 256+16-144(%rdx),%xmm2 + pand %xmm5,%xmm1 + movdqu 256+16-144(%r8),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,256-144(%rcx) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,272-144(%rcx) + .byte 0xf3,0xc3 + +.globl vec_prefetch + +.def vec_prefetch; .scl 2; .type 32; .endef +.p2align 5 +vec_prefetch: + .byte 0xf3,0x0f,0x1e,0xfa + + leaq -1(%rcx,%rdx,1),%rdx + movq $64,%rax + xorq %r8,%r8 + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + cmovaq %r8,%rax + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + cmovaq %r8,%rax + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + cmovaq %r8,%rax + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + cmovaq %r8,%rax + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + cmovaq %r8,%rax + prefetchnta (%rcx) + leaq (%rcx,%rax,1),%rcx + cmpq %rdx,%rcx + cmovaq %rdx,%rcx + prefetchnta (%rcx) + .byte 0xf3,0xc3 + +.globl vec_is_zero_16x + +.def vec_is_zero_16x; .scl 2; .type 32; .endef +.p2align 5 +vec_is_zero_16x: + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%edx + movdqu (%rcx),%xmm0 + leaq 16(%rcx),%rcx + +.Loop_is_zero: + decl %edx + jz .Loop_is_zero_done + movdqu (%rcx),%xmm1 + leaq 16(%rcx),%rcx + por %xmm1,%xmm0 + jmp .Loop_is_zero + +.Loop_is_zero_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %edx + testq %rax,%rax + cmovnzl %edx,%eax + xorl $1,%eax + .byte 0xf3,0xc3 + +.globl vec_is_equal_16x + +.def vec_is_equal_16x; .scl 2; .type 32; .endef +.p2align 5 +vec_is_equal_16x: + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%r8d + movdqu (%rcx),%xmm0 + movdqu (%rdx),%xmm1 + subq %rcx,%rdx + leaq 16(%rcx),%rcx + pxor %xmm1,%xmm0 + +.Loop_is_equal: + decl %r8d + jz .Loop_is_equal_done + movdqu (%rcx),%xmm1 + movdqu (%rcx,%rdx,1),%xmm2 + leaq 16(%rcx),%rcx + pxor %xmm2,%xmm1 + por %xmm1,%xmm0 + jmp .Loop_is_equal + +.Loop_is_equal_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %r8d + testq %rax,%rax + cmovnzl %r8d,%eax + xorl $1,%eax + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_add_mod_384 +.rva .LSEH_body_add_mod_384 +.rva .LSEH_info_add_mod_384_prologue + +.rva .LSEH_body_add_mod_384 +.rva .LSEH_epilogue_add_mod_384 +.rva .LSEH_info_add_mod_384_body + +.rva .LSEH_epilogue_add_mod_384 +.rva .LSEH_end_add_mod_384 +.rva .LSEH_info_add_mod_384_epilogue + +.rva .LSEH_begin_add_mod_384x +.rva .LSEH_body_add_mod_384x +.rva .LSEH_info_add_mod_384x_prologue + +.rva .LSEH_body_add_mod_384x +.rva .LSEH_epilogue_add_mod_384x +.rva .LSEH_info_add_mod_384x_body + +.rva .LSEH_epilogue_add_mod_384x +.rva .LSEH_end_add_mod_384x +.rva .LSEH_info_add_mod_384x_epilogue + +.rva .LSEH_begin_rshift_mod_384 +.rva .LSEH_body_rshift_mod_384 +.rva .LSEH_info_rshift_mod_384_prologue + +.rva .LSEH_body_rshift_mod_384 +.rva .LSEH_epilogue_rshift_mod_384 +.rva .LSEH_info_rshift_mod_384_body + +.rva .LSEH_epilogue_rshift_mod_384 +.rva .LSEH_end_rshift_mod_384 +.rva .LSEH_info_rshift_mod_384_epilogue + +.rva .LSEH_begin_div_by_2_mod_384 +.rva .LSEH_body_div_by_2_mod_384 +.rva .LSEH_info_div_by_2_mod_384_prologue + +.rva .LSEH_body_div_by_2_mod_384 +.rva .LSEH_epilogue_div_by_2_mod_384 +.rva .LSEH_info_div_by_2_mod_384_body + +.rva .LSEH_epilogue_div_by_2_mod_384 +.rva .LSEH_end_div_by_2_mod_384 +.rva .LSEH_info_div_by_2_mod_384_epilogue + +.rva .LSEH_begin_lshift_mod_384 +.rva .LSEH_body_lshift_mod_384 +.rva .LSEH_info_lshift_mod_384_prologue + +.rva .LSEH_body_lshift_mod_384 +.rva .LSEH_epilogue_lshift_mod_384 +.rva .LSEH_info_lshift_mod_384_body + +.rva .LSEH_epilogue_lshift_mod_384 +.rva .LSEH_end_lshift_mod_384 +.rva .LSEH_info_lshift_mod_384_epilogue + +.rva .LSEH_begin_mul_by_3_mod_384 +.rva .LSEH_body_mul_by_3_mod_384 +.rva .LSEH_info_mul_by_3_mod_384_prologue + +.rva .LSEH_body_mul_by_3_mod_384 +.rva .LSEH_epilogue_mul_by_3_mod_384 +.rva .LSEH_info_mul_by_3_mod_384_body + +.rva .LSEH_epilogue_mul_by_3_mod_384 +.rva .LSEH_end_mul_by_3_mod_384 +.rva .LSEH_info_mul_by_3_mod_384_epilogue + +.rva .LSEH_begin_mul_by_8_mod_384 +.rva .LSEH_body_mul_by_8_mod_384 +.rva .LSEH_info_mul_by_8_mod_384_prologue + +.rva .LSEH_body_mul_by_8_mod_384 +.rva .LSEH_epilogue_mul_by_8_mod_384 +.rva .LSEH_info_mul_by_8_mod_384_body + +.rva .LSEH_epilogue_mul_by_8_mod_384 +.rva .LSEH_end_mul_by_8_mod_384 +.rva .LSEH_info_mul_by_8_mod_384_epilogue + +.rva .LSEH_begin_mul_by_3_mod_384x +.rva .LSEH_body_mul_by_3_mod_384x +.rva .LSEH_info_mul_by_3_mod_384x_prologue + +.rva .LSEH_body_mul_by_3_mod_384x +.rva .LSEH_epilogue_mul_by_3_mod_384x +.rva .LSEH_info_mul_by_3_mod_384x_body + +.rva .LSEH_epilogue_mul_by_3_mod_384x +.rva .LSEH_end_mul_by_3_mod_384x +.rva .LSEH_info_mul_by_3_mod_384x_epilogue + +.rva .LSEH_begin_mul_by_8_mod_384x +.rva .LSEH_body_mul_by_8_mod_384x +.rva .LSEH_info_mul_by_8_mod_384x_prologue + +.rva .LSEH_body_mul_by_8_mod_384x +.rva .LSEH_epilogue_mul_by_8_mod_384x +.rva .LSEH_info_mul_by_8_mod_384x_body + +.rva .LSEH_epilogue_mul_by_8_mod_384x +.rva .LSEH_end_mul_by_8_mod_384x +.rva .LSEH_info_mul_by_8_mod_384x_epilogue + +.rva .LSEH_begin_cneg_mod_384 +.rva .LSEH_body_cneg_mod_384 +.rva .LSEH_info_cneg_mod_384_prologue + +.rva .LSEH_body_cneg_mod_384 +.rva .LSEH_epilogue_cneg_mod_384 +.rva .LSEH_info_cneg_mod_384_body + +.rva .LSEH_epilogue_cneg_mod_384 +.rva .LSEH_end_cneg_mod_384 +.rva .LSEH_info_cneg_mod_384_epilogue + +.rva .LSEH_begin_sub_mod_384 +.rva .LSEH_body_sub_mod_384 +.rva .LSEH_info_sub_mod_384_prologue + +.rva .LSEH_body_sub_mod_384 +.rva .LSEH_epilogue_sub_mod_384 +.rva .LSEH_info_sub_mod_384_body + +.rva .LSEH_epilogue_sub_mod_384 +.rva .LSEH_end_sub_mod_384 +.rva .LSEH_info_sub_mod_384_epilogue + +.rva .LSEH_begin_sub_mod_384x +.rva .LSEH_body_sub_mod_384x +.rva .LSEH_info_sub_mod_384x_prologue + +.rva .LSEH_body_sub_mod_384x +.rva .LSEH_epilogue_sub_mod_384x +.rva .LSEH_info_sub_mod_384x_body + +.rva .LSEH_epilogue_sub_mod_384x +.rva .LSEH_end_sub_mod_384x +.rva .LSEH_info_sub_mod_384x_epilogue + +.rva .LSEH_begin_mul_by_1_plus_i_mod_384x +.rva .LSEH_body_mul_by_1_plus_i_mod_384x +.rva .LSEH_info_mul_by_1_plus_i_mod_384x_prologue + +.rva .LSEH_body_mul_by_1_plus_i_mod_384x +.rva .LSEH_epilogue_mul_by_1_plus_i_mod_384x +.rva .LSEH_info_mul_by_1_plus_i_mod_384x_body + +.rva .LSEH_epilogue_mul_by_1_plus_i_mod_384x +.rva .LSEH_end_mul_by_1_plus_i_mod_384x +.rva .LSEH_info_mul_by_1_plus_i_mod_384x_epilogue + +.rva .LSEH_begin_sgn0_pty_mod_384 +.rva .LSEH_body_sgn0_pty_mod_384 +.rva .LSEH_info_sgn0_pty_mod_384_prologue + +.rva .LSEH_body_sgn0_pty_mod_384 +.rva .LSEH_epilogue_sgn0_pty_mod_384 +.rva .LSEH_info_sgn0_pty_mod_384_body + +.rva .LSEH_epilogue_sgn0_pty_mod_384 +.rva .LSEH_end_sgn0_pty_mod_384 +.rva .LSEH_info_sgn0_pty_mod_384_epilogue + +.rva .LSEH_begin_sgn0_pty_mod_384x +.rva .LSEH_body_sgn0_pty_mod_384x +.rva .LSEH_info_sgn0_pty_mod_384x_prologue + +.rva .LSEH_body_sgn0_pty_mod_384x +.rva .LSEH_epilogue_sgn0_pty_mod_384x +.rva .LSEH_info_sgn0_pty_mod_384x_body + +.rva .LSEH_epilogue_sgn0_pty_mod_384x +.rva .LSEH_end_sgn0_pty_mod_384x +.rva .LSEH_info_sgn0_pty_mod_384x_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_add_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_add_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_add_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_add_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_add_mod_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x03,0x00 +.byte 0x00,0xe4,0x04,0x00 +.byte 0x00,0xd4,0x05,0x00 +.byte 0x00,0xc4,0x06,0x00 +.byte 0x00,0x34,0x07,0x00 +.byte 0x00,0x54,0x08,0x00 +.byte 0x00,0x74,0x0a,0x00 +.byte 0x00,0x64,0x0b,0x00 +.byte 0x00,0x82 +.byte 0x00,0x00 +.LSEH_info_add_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_rshift_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_rshift_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_rshift_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_div_by_2_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_div_by_2_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_div_by_2_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_lshift_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_lshift_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_lshift_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_3_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_3_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mul_by_3_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_8_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_8_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mul_by_8_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_3_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_3_mod_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mul_by_3_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_8_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_8_mod_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mul_by_8_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_cneg_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_cneg_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_cneg_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sub_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sub_mod_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sub_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sub_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sub_mod_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x03,0x00 +.byte 0x00,0xe4,0x04,0x00 +.byte 0x00,0xd4,0x05,0x00 +.byte 0x00,0xc4,0x06,0x00 +.byte 0x00,0x34,0x07,0x00 +.byte 0x00,0x54,0x08,0x00 +.byte 0x00,0x74,0x0a,0x00 +.byte 0x00,0x64,0x0b,0x00 +.byte 0x00,0x82 +.byte 0x00,0x00 +.LSEH_info_sub_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_by_1_plus_i_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_by_1_plus_i_mod_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x07,0x00 +.byte 0x00,0xe4,0x08,0x00 +.byte 0x00,0xd4,0x09,0x00 +.byte 0x00,0xc4,0x0a,0x00 +.byte 0x00,0x34,0x0b,0x00 +.byte 0x00,0x54,0x0c,0x00 +.byte 0x00,0x74,0x0e,0x00 +.byte 0x00,0x64,0x0f,0x00 +.byte 0x00,0xc2 +.byte 0x00,0x00 +.LSEH_info_mul_by_1_plus_i_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0_pty_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0_pty_mod_384_body: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 +.LSEH_info_sgn0_pty_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0_pty_mod_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0_pty_mod_384x_body: +.byte 1,0,9,0 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00 +.LSEH_info_sgn0_pty_mod_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/add_mod_384x384-x86_64.s b/crypto/blst_src/build/coff/add_mod_384x384-x86_64.s new file mode 100644 index 00000000000..79976cc0e7a --- /dev/null +++ b/crypto/blst_src/build/coff/add_mod_384x384-x86_64.s @@ -0,0 +1,326 @@ +.text + +.def __add_mod_384x384; .scl 3; .type 32; .endef +.p2align 5 +__add_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + addq 0(%rdx),%r8 + movq 56(%rsi),%r15 + adcq 8(%rdx),%r9 + movq 64(%rsi),%rax + adcq 16(%rdx),%r10 + movq 72(%rsi),%rbx + adcq 24(%rdx),%r11 + movq 80(%rsi),%rbp + adcq 32(%rdx),%r12 + movq 88(%rsi),%rsi + adcq 40(%rdx),%r13 + movq %r8,0(%rdi) + adcq 48(%rdx),%r14 + movq %r9,8(%rdi) + adcq 56(%rdx),%r15 + movq %r10,16(%rdi) + adcq 64(%rdx),%rax + movq %r12,32(%rdi) + movq %r14,%r8 + adcq 72(%rdx),%rbx + movq %r11,24(%rdi) + movq %r15,%r9 + adcq 80(%rdx),%rbp + movq %r13,40(%rdi) + movq %rax,%r10 + adcq 88(%rdx),%rsi + movq %rbx,%r11 + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %rbp,%r12 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%rbx + sbbq 32(%rcx),%rbp + movq %rsi,%r13 + sbbq 40(%rcx),%rsi + sbbq $0,%rdx + + cmovcq %r8,%r14 + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %r14,48(%rdi) + cmovcq %r11,%rbx + movq %r15,56(%rdi) + cmovcq %r12,%rbp + movq %rax,64(%rdi) + cmovcq %r13,%rsi + movq %rbx,72(%rdi) + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 + + +.def __sub_mod_384x384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 + + +.globl add_mod_384x384 + +.def add_mod_384x384; .scl 2; .type 32; .endef +.p2align 5 +add_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_add_mod_384x384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_add_mod_384x384: + + + call __add_mod_384x384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_add_mod_384x384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_add_mod_384x384: + +.globl sub_mod_384x384 + +.def sub_mod_384x384; .scl 2; .type 32; .endef +.p2align 5 +sub_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sub_mod_384x384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sub_mod_384x384: + + + call __sub_mod_384x384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sub_mod_384x384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sub_mod_384x384: +.section .pdata +.p2align 2 +.rva .LSEH_begin_add_mod_384x384 +.rva .LSEH_body_add_mod_384x384 +.rva .LSEH_info_add_mod_384x384_prologue + +.rva .LSEH_body_add_mod_384x384 +.rva .LSEH_epilogue_add_mod_384x384 +.rva .LSEH_info_add_mod_384x384_body + +.rva .LSEH_epilogue_add_mod_384x384 +.rva .LSEH_end_add_mod_384x384 +.rva .LSEH_info_add_mod_384x384_epilogue + +.rva .LSEH_begin_sub_mod_384x384 +.rva .LSEH_body_sub_mod_384x384 +.rva .LSEH_info_sub_mod_384x384_prologue + +.rva .LSEH_body_sub_mod_384x384 +.rva .LSEH_epilogue_sub_mod_384x384 +.rva .LSEH_info_sub_mod_384x384_body + +.rva .LSEH_epilogue_sub_mod_384x384 +.rva .LSEH_end_sub_mod_384x384 +.rva .LSEH_info_sub_mod_384x384_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_add_mod_384x384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_add_mod_384x384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_add_mod_384x384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sub_mod_384x384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sub_mod_384x384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sub_mod_384x384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/ct_inverse_mod_256-armv8.S b/crypto/blst_src/build/coff/ct_inverse_mod_256-armv8.S new file mode 100644 index 00000000000..17c3d25278f --- /dev/null +++ b/crypto/blst_src/build/coff/ct_inverse_mod_256-armv8.S @@ -0,0 +1,798 @@ +.text + +.globl ct_inverse_mod_256 +.def ct_inverse_mod_256; +.type 32; +.endef +.p2align 5 +ct_inverse_mod_256: +.long 3573752639 + stp x29, x30, [sp,#-80]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + sub sp, sp, #1040 + + ldp x4, x5, [x1,#8*0] + ldp x6, x7, [x1,#8*2] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + str x0, [sp] + + ldp x8, x9, [x2,#8*0] + ldp x10, x11, [x2,#8*2] + + stp x4, x5, [x1,#8*0] // copy input to |a| + stp x6, x7, [x1,#8*2] + stp x8, x9, [x1,#8*4] // copy modulus to |b| + stp x10, x11, [x1,#8*6] + + ////////////////////////////////////////// first iteration + bl .Lab_approximation_31_256_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + str x12,[x0,#8*8] // initialize |u| with |f0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to dst |b| + bl __smul_256_n_shift_by_31 + str x12, [x0,#8*9] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + ldr x8, [x1,#8*8] // |u| + ldr x9, [x1,#8*13] // |v| + madd x4, x16, x8, xzr // |u|*|f0| + madd x4, x17, x9, x4 // |v|*|g0| + str x4, [x0,#8*4] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*5] + stp x5, x5, [x0,#8*7] + + madd x4, x12, x8, xzr // |u|*|f1| + madd x4, x13, x9, x4 // |v|*|g1| + str x4, [x0,#8*9] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*10] + stp x5, x5, [x0,#8*12] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + ////////////////////////////////////////// two[!] last iterations + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #47 // 31 + 512 % 31 + //bl __ab_approximation_62_256 // |a| and |b| are exact, + ldr x7, [x1,#8*0] // just load + ldr x11, [x1,#8*4] + bl __inner_loop_62_256 + + mov x16, x14 + mov x17, x15 + ldr x0, [sp] // original out_ptr + bl __smul_256x63 + bl __smul_512x63_tail + ldr x30, [x29,#8] + + smulh x20, x7, x17 // figure out top-most limb + ldp x8, x9, [x3,#8*0] + adc x23, x23, x25 + ldp x10, x11, [x3,#8*2] + + add x20, x20, x23 // x20 is 1, 0 or -1 + asr x19, x20, #63 // sign as mask + + and x23, x8, x19 // add mod<<256 conditionally + and x24, x9, x19 + adds x4, x4, x23 + and x25, x10, x19 + adcs x5, x5, x24 + and x26, x11, x19 + adcs x6, x6, x25 + adcs x7, x22, x26 + adc x20, x20, xzr // x20 is 1, 0 or -1 + + neg x19, x20 + orr x20, x20, x19 // excess bit or sign as mask + asr x19, x19, #63 // excess bit as mask + + and x8, x8, x20 // mask |mod| + and x9, x9, x20 + and x10, x10, x20 + and x11, x11, x20 + + eor x8, x8, x19 // conditionally negate |mod| + eor x9, x9, x19 + adds x8, x8, x19, lsr#63 + eor x10, x10, x19 + adcs x9, x9, xzr + eor x11, x11, x19 + adcs x10, x10, xzr + adc x11, x11, xzr + + adds x4, x4, x8 // final adjustment for |mod|<<256 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*4] + adc x7, x7, x11 + stp x6, x7, [x0,#8*6] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldr x29, [sp],#80 +.long 3573752767 + ret + + +//////////////////////////////////////////////////////////////////////// +.def __smul_256x63; +.type 32; +.endef +.p2align 5 +__smul_256x63: + ldp x4, x5, [x1,#8*0+64] // load |u| (or |v|) + asr x14, x16, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x6, x7, [x1,#8*2+64] + eor x16, x16, x14 // conditionally negate |f_| (or |g_|) + ldr x22, [x1,#8*4+64] + + eor x4, x4, x14 // conditionally negate |u| (or |v|) + sub x16, x16, x14 + eor x5, x5, x14 + adds x4, x4, x14, lsr#63 + eor x6, x6, x14 + adcs x5, x5, xzr + eor x7, x7, x14 + adcs x6, x6, xzr + eor x22, x22, x14 + umulh x19, x4, x16 + adcs x7, x7, xzr + umulh x20, x5, x16 + adcs x22, x22, xzr + umulh x21, x6, x16 + mul x4, x4, x16 + cmp x16, #0 + mul x5, x5, x16 + csel x22, x22, xzr, ne + mul x6, x6, x16 + adds x5, x5, x19 + mul x24, x7, x16 + adcs x6, x6, x20 + adcs x24, x24, x21 + adc x26, xzr, xzr + ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x10, x11, [x1,#8*2+104] + eor x17, x17, x14 // conditionally negate |f_| (or |g_|) + ldr x23, [x1,#8*4+104] + + eor x8, x8, x14 // conditionally negate |u| (or |v|) + sub x17, x17, x14 + eor x9, x9, x14 + adds x8, x8, x14, lsr#63 + eor x10, x10, x14 + adcs x9, x9, xzr + eor x11, x11, x14 + adcs x10, x10, xzr + eor x23, x23, x14 + umulh x19, x8, x17 + adcs x11, x11, xzr + umulh x20, x9, x17 + adcs x23, x23, xzr + umulh x21, x10, x17 + adc x15, xzr, xzr // used in __smul_512x63_tail + mul x8, x8, x17 + cmp x17, #0 + mul x9, x9, x17 + csel x23, x23, xzr, ne + mul x10, x10, x17 + adds x9, x9, x19 + mul x25, x11, x17 + adcs x10, x10, x20 + adcs x25, x25, x21 + adc x26, x26, xzr + + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*0] + adcs x24, x24, x25 + stp x6, x24, [x0,#8*2] + + ret + + +.def __smul_512x63_tail; +.type 32; +.endef +.p2align 5 +__smul_512x63_tail: + umulh x24, x7, x16 + ldp x5, x6, [x1,#8*18] // load rest of |v| + adc x26, x26, xzr + ldr x7, [x1,#8*20] + and x22, x22, x16 + + umulh x11, x11, x17 // resume |v|*|g1| chain + + sub x24, x24, x22 // tie up |u|*|f1| chain + asr x25, x24, #63 + + eor x5, x5, x14 // conditionally negate rest of |v| + eor x6, x6, x14 + adds x5, x5, x15 + eor x7, x7, x14 + adcs x6, x6, xzr + umulh x19, x23, x17 + adc x7, x7, xzr + umulh x20, x5, x17 + add x11, x11, x26 + umulh x21, x6, x17 + + mul x4, x23, x17 + mul x5, x5, x17 + adds x4, x4, x11 + mul x6, x6, x17 + adcs x5, x5, x19 + mul x22, x7, x17 + adcs x6, x6, x20 + adcs x22, x22, x21 + adc x23, xzr, xzr // used in the final step + + adds x4, x4, x24 + adcs x5, x5, x25 + adcs x6, x6, x25 + stp x4, x5, [x0,#8*4] + adcs x22, x22, x25 // carry is used in the final step + stp x6, x22, [x0,#8*6] + + ret + + +.def __smul_256_n_shift_by_31; +.type 32; +.endef +.p2align 5 +__smul_256_n_shift_by_31: + ldp x4, x5, [x1,#8*0+0] // load |a| (or |b|) + asr x24, x12, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x6, x7, [x1,#8*2+0] + eor x25, x12, x24 // conditionally negate |f0| (or |g0|) + + eor x4, x4, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x5, x5, x24 + adds x4, x4, x24, lsr#63 + eor x6, x6, x24 + adcs x5, x5, xzr + eor x7, x7, x24 + umulh x19, x4, x25 + adcs x6, x6, xzr + umulh x20, x5, x25 + adc x7, x7, xzr + umulh x21, x6, x25 + and x24, x24, x25 + umulh x22, x7, x25 + neg x24, x24 + + mul x4, x4, x25 + mul x5, x5, x25 + mul x6, x6, x25 + adds x5, x5, x19 + mul x7, x7, x25 + adcs x6, x6, x20 + adcs x7, x7, x21 + adc x22, x22, x24 + ldp x8, x9, [x1,#8*0+32] // load |a| (or |b|) + asr x24, x13, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x10, x11, [x1,#8*2+32] + eor x25, x13, x24 // conditionally negate |f0| (or |g0|) + + eor x8, x8, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x9, x9, x24 + adds x8, x8, x24, lsr#63 + eor x10, x10, x24 + adcs x9, x9, xzr + eor x11, x11, x24 + umulh x19, x8, x25 + adcs x10, x10, xzr + umulh x20, x9, x25 + adc x11, x11, xzr + umulh x21, x10, x25 + and x24, x24, x25 + umulh x23, x11, x25 + neg x24, x24 + + mul x8, x8, x25 + mul x9, x9, x25 + mul x10, x10, x25 + adds x9, x9, x19 + mul x11, x11, x25 + adcs x10, x10, x20 + adcs x11, x11, x21 + adc x23, x23, x24 + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + adcs x7, x7, x11 + adc x8, x22, x23 + + extr x4, x5, x4, #31 + extr x5, x6, x5, #31 + extr x6, x7, x6, #31 + asr x23, x8, #63 // result's sign as mask + extr x7, x8, x7, #31 + + eor x4, x4, x23 // ensure the result is positive + eor x5, x5, x23 + adds x4, x4, x23, lsr#63 + eor x6, x6, x23 + adcs x5, x5, xzr + eor x7, x7, x23 + adcs x6, x6, xzr + stp x4, x5, [x0,#8*0] + adc x7, x7, xzr + stp x6, x7, [x0,#8*2] + + eor x12, x12, x23 // adjust |f/g| accordingly + eor x13, x13, x23 + sub x12, x12, x23 + sub x13, x13, x23 + + ret + +.def __ab_approximation_31_256; +.type 32; +.endef +.p2align 4 +__ab_approximation_31_256: + ldp x6, x7, [x1,#8*2] + ldp x10, x11, [x1,#8*6] + ldp x4, x5, [x1,#8*0] + ldp x8, x9, [x1,#8*4] + +.Lab_approximation_31_256_loaded: + orr x19, x7, x11 // check top-most limbs, ... + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x5, ne + orr x19, x7, x11 // and ones before top-most, ... + csel x10, x10, x9, ne + + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x4, ne + orr x19, x7, x11 // and one more, ... + csel x10, x10, x8, ne + + clz x19, x19 + cmp x19, #64 + csel x19, x19, xzr, ne + csel x7, x7, x6, ne + csel x11, x11, x10, ne + neg x20, x19 + + lslv x7, x7, x19 // align high limbs to the left + lslv x11, x11, x19 + lsrv x6, x6, x20 + lsrv x10, x10, x20 + and x6, x6, x20, asr#6 + and x10, x10, x20, asr#6 + orr x7, x7, x6 + orr x11, x11, x10 + + bfxil x7, x4, #0, #31 + bfxil x11, x8, #0, #31 + + b __inner_loop_31_256 + ret + + +.def __inner_loop_31_256; +.type 32; +.endef +.p2align 4 +__inner_loop_31_256: + mov x2, #31 + mov x13, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x15, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x23,#0x7FFFFFFF7FFFFFFF + +.Loop_31_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x15 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x15, x15, x13, hs // exchange |fg0| and |fg1| + csel x13, x13, x19, hs + lsr x7, x7, #1 + and x19, x15, x22 + and x20, x23, x22 + sub x13, x13, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x15, x15, x15 // |f1|<<=1 + add x13, x13, x20 + sub x15, x15, x23 + cbnz x2, .Loop_31_256 + + mov x23, #0x7FFFFFFF + ubfx x12, x13, #0, #32 + ubfx x13, x13, #32, #32 + ubfx x14, x15, #0, #32 + ubfx x15, x15, #32, #32 + sub x12, x12, x23 // remove bias + sub x13, x13, x23 + sub x14, x14, x23 + sub x15, x15, x23 + + ret + + +.def __inner_loop_62_256; +.type 32; +.endef +.p2align 4 +__inner_loop_62_256: + mov x12, #1 // |f0|=1 + mov x13, #0 // |g0|=0 + mov x14, #0 // |f1|=0 + mov x15, #1 // |g1|=1 + +.Loop_62_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x12 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + mov x20, x13 + csel x12, x12, x14, hs // exchange |f0| and |f1| + csel x14, x14, x19, hs + csel x13, x13, x15, hs // exchange |g0| and |g1| + csel x15, x15, x20, hs + lsr x7, x7, #1 + and x19, x14, x22 + and x20, x15, x22 + add x14, x14, x14 // |f1|<<=1 + add x15, x15, x15 // |g1|<<=1 + sub x12, x12, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x13, x13, x20 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62_256 + + ret + diff --git a/crypto/blst_src/build/coff/ct_inverse_mod_256-x86_64.s b/crypto/blst_src/build/coff/ct_inverse_mod_256-x86_64.s new file mode 100644 index 00000000000..e7d4a6313b1 --- /dev/null +++ b/crypto/blst_src/build/coff/ct_inverse_mod_256-x86_64.s @@ -0,0 +1,1209 @@ +.text + +.globl ct_inverse_mod_256 +.def ct_inverse_mod_256; .scl 2; .type 32; .endef +.p2align 5 +ct_inverse_mod_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_ct_inverse_mod_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $1072,%rsp + +.LSEH_body_ct_inverse_mod_256: + + + leaq 48+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + movq 0(%rdx),%r12 + movq 8(%rdx),%r13 + movq 16(%rdx),%r14 + movq 24(%rdx),%r15 + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + + movq %r12,32(%rax) + movq %r13,40(%rax) + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rax,%rsi + + + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,64(%rdi) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,72(%rdi) + + + xorq $256,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + + movq 64(%rsi),%r8 + movq 104(%rsi),%r12 + movq %r8,%r9 + imulq 0(%rsp),%r8 + movq %r12,%r13 + imulq 8(%rsp),%r12 + addq %r12,%r8 + movq %r8,32(%rdi) + sarq $63,%r8 + movq %r8,40(%rdi) + movq %r8,48(%rdi) + movq %r8,56(%rdi) + movq %r8,64(%rdi) + leaq 64(%rsi),%rsi + + imulq %rdx,%r9 + imulq %rcx,%r13 + addq %r13,%r9 + movq %r9,72(%rdi) + sarq $63,%r9 + movq %r9,80(%rdi) + movq %r9,88(%rdi) + movq %r9,96(%rdi) + movq %r9,104(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + sarq $63,%rbp + movq %rbp,40(%rdi) + movq %rbp,48(%rdi) + movq %rbp,56(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + + xorq $256+64,%rsi + movl $47,%edx + + movq 0(%rsi),%r8 + + movq 32(%rsi),%r10 + + call __inner_loop_62_256 + + + + + + + + leaq 64(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_512x63 + adcq %rbp,%rdx + + movq 40(%rsp),%rsi + movq %rdx,%rax + sarq $63,%rdx + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + adcq $0,%rax + + movq %rax,%rdx + negq %rax + orq %rax,%rdx + sarq $63,%rax + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + xorq %rax,%r8 + xorq %rcx,%rcx + xorq %rax,%r9 + subq %rax,%rcx + xorq %rax,%r10 + xorq %rax,%rdx + addq %rcx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + leaq 1072(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_ct_inverse_mod_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_ct_inverse_mod_256: +.def __smulq_512x63; .scl 3; .type 32; .endef +.p2align 5 +__smulq_512x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %r9,8(%rdi) + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %r10,16(%rdi) + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %r11,24(%rdi) + + movq 40(%rsi),%r8 + movq 48(%rsi),%r9 + movq 56(%rsi),%r10 + movq 64(%rsi),%r11 + movq 72(%rsi),%r12 + movq 80(%rsi),%r13 + movq 88(%rsi),%r14 + movq 96(%rsi),%r15 + + movq %rcx,%rdx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rcx + addq %rax,%rcx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rcx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rcx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rcx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rcx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rcx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rcx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rcx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + imulq %rcx + addq %rax,%r15 + adcq $0,%rdx + + movq %rbp,%rbx + sarq $63,%rbp + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq %rbx,%r12 + adcq %rbp,%r13 + adcq %rbp,%r14 + adcq %rbp,%r15 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + .byte 0xf3,0xc3 + + +.def __smulq_256x63; .scl 3; .type 32; .endef +.p2align 5 +__smulq_256x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %rcx,%rdx + movq 40+0(%rsi),%r12 + movq 40+8(%rsi),%r13 + movq 40+16(%rsi),%r14 + movq 40+24(%rsi),%r15 + movq 40+32(%rsi),%rcx + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rcx + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rcx + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + andq %rbx,%rcx + negq %rcx + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %rbp,32(%rdi) + + .byte 0xf3,0xc3 + +.def __smulq_256_n_shift_by_31; .scl 3; .type 32; .endef +.p2align 5 +__smulq_256_n_shift_by_31: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,0(%rdi) + movq %rcx,8(%rdi) + movq %rdx,%rbp + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + + movq %rbp,%rbx + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rbx + addq %rax,%rbx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + andq %rbx,%rbp + negq %rbp + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r14 + movq 32+24(%rsi),%r15 + + movq %rcx,%rbx + sarq $63,%rcx + xorq %rax,%rax + subq %rcx,%rax + + xorq %rcx,%rbx + addq %rax,%rbx + + xorq %rcx,%r12 + xorq %rcx,%r13 + xorq %rcx,%r14 + xorq %rcx,%r15 + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + andq %rbx,%rcx + negq %rcx + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq 0(%rdi),%rdx + movq 8(%rdi),%rcx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%rbp,%r11 + + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + xorq %rbp,%rdx + xorq %rbp,%rcx + addq %rax,%rdx + addq %rax,%rcx + + .byte 0xf3,0xc3 + +.def __ab_approximation_31_256; .scl 3; .type 32; .endef +.p2align 5 +__ab_approximation_31_256: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 24(%rsi),%r9 + movq 56(%rsi),%r11 + movq 16(%rsi),%rbx + movq 48(%rsi),%rbp + movq 8(%rsi),%r8 + movq 40(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 32(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + notq %rax + andq %rax,%r9 + andq %rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31_256 + + .byte 0xf3,0xc3 + +.def __inner_loop_31_256; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_31_256: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +.Loop_31_256: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edx + jnz .Loop_31_256 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 + + +.def __inner_loop_62_256; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_62_256: + .byte 0xf3,0x0f,0x1e,0xfa + + movl %edx,%r15d + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq %rdx,%r13 + movq %rdx,%r14 + +.Loop_62_256: + xorq %rax,%rax + testq %r14,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq %r14,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%r15d + jnz .Loop_62_256 + + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_ct_inverse_mod_256 +.rva .LSEH_body_ct_inverse_mod_256 +.rva .LSEH_info_ct_inverse_mod_256_prologue + +.rva .LSEH_body_ct_inverse_mod_256 +.rva .LSEH_epilogue_ct_inverse_mod_256 +.rva .LSEH_info_ct_inverse_mod_256_body + +.rva .LSEH_epilogue_ct_inverse_mod_256 +.rva .LSEH_end_ct_inverse_mod_256 +.rva .LSEH_info_ct_inverse_mod_256_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_ct_inverse_mod_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_ct_inverse_mod_256_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x86,0x00 +.byte 0x00,0xe4,0x87,0x00 +.byte 0x00,0xd4,0x88,0x00 +.byte 0x00,0xc4,0x89,0x00 +.byte 0x00,0x34,0x8a,0x00 +.byte 0x00,0x54,0x8b,0x00 +.byte 0x00,0x74,0x8d,0x00 +.byte 0x00,0x64,0x8e,0x00 +.byte 0x00,0x01,0x8c,0x00 +.LSEH_info_ct_inverse_mod_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/ct_inverse_mod_384-armv8.S b/crypto/blst_src/build/coff/ct_inverse_mod_384-armv8.S new file mode 100644 index 00000000000..65193f1e96a --- /dev/null +++ b/crypto/blst_src/build/coff/ct_inverse_mod_384-armv8.S @@ -0,0 +1,729 @@ +.text + +.globl ct_inverse_mod_383 +.def ct_inverse_mod_383; +.type 32; +.endef +.p2align 5 +ct_inverse_mod_383: +.long 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #1040 + + ldp x22, x4, [x1,#8*0] + ldp x5, x6, [x1,#8*2] + ldp x7, x8, [x1,#8*4] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + stp x0, x3, [sp] + + ldp x9, x10, [x2,#8*0] + ldp x11, x12, [x2,#8*2] + ldp x13, x14, [x2,#8*4] + + stp x22, x4, [x1,#8*0] // copy input to |a| + stp x5, x6, [x1,#8*2] + stp x7, x8, [x1,#8*4] + stp x9, x10, [x1,#8*6] // copy modulus to |b| + stp x11, x12, [x1,#8*8] + stp x13, x14, [x1,#8*10] + + ////////////////////////////////////////// first iteration + mov x2, #62 + bl .Lab_approximation_62_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + str x15,[x0,#8*12] // initialize |u| with |f0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to dst |b| + bl __smul_383_n_shift_by_62 + str x15, [x0,#8*12] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + ldr x7, [x1,#8*12] // |u| + ldr x8, [x1,#8*18] // |v| + mul x3, x20, x7 // |u|*|f0| + smulh x4, x20, x7 + mul x5, x21, x8 // |v|*|g0| + smulh x6, x21, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*6] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*8] + stp x5, x5, [x0,#8*10] + + mul x3, x15, x7 // |u|*|f1| + smulh x4, x15, x7 + mul x5, x16, x8 // |v|*|g1| + smulh x6, x16, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*12] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*14] + stp x5, x5, [x0,#8*16] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + asr x27, x27, #63 // sign extension + stp x27, x27, [x0,#8*6] + stp x27, x27, [x0,#8*8] + stp x27, x27, [x0,#8*10] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + ////////////////////////////////////////// iteration before last + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldp x3, x8, [x1,#8*0] // just load + ldp x9, x14, [x1,#8*6] + bl __inner_loop_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + str x3, [x0,#8*0] + str x9, [x0,#8*6] + + mov x20, x15 // exact |f0| + mov x21, x16 // exact |g0| + mov x15, x17 + mov x16, x19 + add x0, x0, #8*12 // pointer to dst |u| + bl __smul_383x63 + + mov x20, x15 // exact |f1| + mov x21, x16 // exact |g1| + add x0, x0, #8*6 // pointer to dst |v| + bl __smul_383x63 + bl __smul_767x63_tail + + ////////////////////////////////////////// last iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #22 // 766 % 62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldr x3, [x1,#8*0] // just load + eor x8, x8, x8 + ldr x9, [x1,#8*6] + eor x14, x14, x14 + bl __inner_loop_62 + + mov x20, x17 + mov x21, x19 + ldp x0, x15, [sp] // original out_ptr and n_ptr + bl __smul_383x63 + bl __smul_767x63_tail + ldr x30, [x29,#8] + + asr x22, x8, #63 // sign as mask + ldp x9, x10, [x15,#8*0] + ldp x11, x12, [x15,#8*2] + ldp x13, x14, [x15,#8*4] + + and x9, x9, x22 // add mod<<384 conditionally + and x10, x10, x22 + adds x3, x3, x9 + and x11, x11, x22 + adcs x4, x4, x10 + and x12, x12, x22 + adcs x5, x5, x11 + and x13, x13, x22 + adcs x6, x6, x12 + and x14, x14, x22 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*8] + adc x8, x8, x14 + stp x7, x8, [x0,#8*10] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 +.long 3573752767 + ret + + +//////////////////////////////////////////////////////////////////////// +// see corresponding commentary in ctx_inverse_mod_384-x86_64... +.def __smul_383x63; +.type 32; +.endef +.p2align 5 +__smul_383x63: + ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) + asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x5, x6, [x1,#8*2+96] + eor x20, x20, x17 // conditionally negate |f_| (or |g_|) + ldp x7, x8, [x1,#8*4+96] + + eor x3, x3, x17 // conditionally negate |u| (or |v|) + sub x20, x20, x17 + eor x4, x4, x17 + adds x3, x3, x17, lsr#63 + eor x5, x5, x17 + adcs x4, x4, xzr + eor x6, x6, x17 + adcs x5, x5, xzr + eor x7, x7, x17 + adcs x6, x6, xzr + umulh x22, x3, x20 + eor x8, x8, x17 + umulh x23, x4, x20 + adcs x7, x7, xzr + umulh x24, x5, x20 + adcs x8, x8, xzr + umulh x25, x6, x20 + umulh x26, x7, x20 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x22 + mul x6, x6, x20 + adcs x5, x5, x23 + mul x7, x7, x20 + adcs x6, x6, x24 + mul x27,x8, x20 + adcs x7, x7, x25 + adcs x27,x27,x26 + adc x2, xzr, xzr + ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x11, x12, [x1,#8*2+144] + eor x21, x21, x17 // conditionally negate |f_| (or |g_|) + ldp x13, x14, [x1,#8*4+144] + + eor x9, x9, x17 // conditionally negate |u| (or |v|) + sub x21, x21, x17 + eor x10, x10, x17 + adds x9, x9, x17, lsr#63 + eor x11, x11, x17 + adcs x10, x10, xzr + eor x12, x12, x17 + adcs x11, x11, xzr + eor x13, x13, x17 + adcs x12, x12, xzr + umulh x22, x9, x21 + eor x14, x14, x17 + umulh x23, x10, x21 + adcs x13, x13, xzr + umulh x24, x11, x21 + adcs x14, x14, xzr + umulh x25, x12, x21 + adc x19, xzr, xzr // used in __smul_767x63_tail + umulh x26, x13, x21 + mul x9, x9, x21 + mul x10, x10, x21 + mul x11, x11, x21 + adds x10, x10, x22 + mul x12, x12, x21 + adcs x11, x11, x23 + mul x13, x13, x21 + adcs x12, x12, x24 + mul x28,x14, x21 + adcs x13, x13, x25 + adcs x28,x28,x26 + adc x2, x2, xzr + + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + stp x3, x4, [x0,#8*0] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*2] + adcs x27, x27, x28 + stp x7, x27, [x0,#8*4] + adc x28, x2, xzr // used in __smul_767x63_tail + + ret + + +.def __smul_767x63_tail; +.type 32; +.endef +.p2align 5 +__smul_767x63_tail: + smulh x27, x8, x20 + ldp x3, x4, [x1,#8*24] // load rest of |v| + umulh x14,x14, x21 + ldp x5, x6, [x1,#8*26] + ldp x7, x8, [x1,#8*28] + + eor x3, x3, x17 // conditionally negate rest of |v| + eor x4, x4, x17 + eor x5, x5, x17 + adds x3, x3, x19 + eor x6, x6, x17 + adcs x4, x4, xzr + eor x7, x7, x17 + adcs x5, x5, xzr + eor x8, x8, x17 + adcs x6, x6, xzr + umulh x22, x3, x21 + adcs x7, x7, xzr + umulh x23, x4, x21 + adc x8, x8, xzr + + umulh x24, x5, x21 + add x14, x14, x28 + umulh x25, x6, x21 + asr x28, x27, #63 + umulh x26, x7, x21 + mul x3, x3, x21 + mul x4, x4, x21 + mul x5, x5, x21 + adds x3, x3, x14 + mul x6, x6, x21 + adcs x4, x4, x22 + mul x7, x7, x21 + adcs x5, x5, x23 + mul x8, x8, x21 + adcs x6, x6, x24 + adcs x7, x7, x25 + adc x8, x8, x26 + + adds x3, x3, x27 + adcs x4, x4, x28 + adcs x5, x5, x28 + adcs x6, x6, x28 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x28 + stp x5, x6, [x0,#8*8] + adc x8, x8, x28 + stp x7, x8, [x0,#8*10] + + ret + + +.def __smul_383_n_shift_by_62; +.type 32; +.endef +.p2align 5 +__smul_383_n_shift_by_62: + ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) + asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x5, x6, [x1,#8*2+0] + eor x2, x15, x28 // conditionally negate |f0| (or |g0|) + ldp x7, x8, [x1,#8*4+0] + + eor x3, x3, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + umulh x22, x3, x2 + adcs x6, x6, xzr + umulh x23, x4, x2 + eor x8, x8, x28 + umulh x24, x5, x2 + adcs x7, x7, xzr + umulh x25, x6, x2 + adc x8, x8, xzr + + umulh x26, x7, x2 + smulh x27, x8, x2 + mul x3, x3, x2 + mul x4, x4, x2 + mul x5, x5, x2 + adds x4, x4, x22 + mul x6, x6, x2 + adcs x5, x5, x23 + mul x7, x7, x2 + adcs x6, x6, x24 + mul x8, x8, x2 + adcs x7, x7, x25 + adcs x8, x8 ,x26 + adc x27, x27, xzr + ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) + asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x11, x12, [x1,#8*2+48] + eor x2, x16, x28 // conditionally negate |f0| (or |g0|) + ldp x13, x14, [x1,#8*4+48] + + eor x9, x9, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x10, x10, x28 + adds x9, x9, x28, lsr#63 + eor x11, x11, x28 + adcs x10, x10, xzr + eor x12, x12, x28 + adcs x11, x11, xzr + eor x13, x13, x28 + umulh x22, x9, x2 + adcs x12, x12, xzr + umulh x23, x10, x2 + eor x14, x14, x28 + umulh x24, x11, x2 + adcs x13, x13, xzr + umulh x25, x12, x2 + adc x14, x14, xzr + + umulh x26, x13, x2 + smulh x28, x14, x2 + mul x9, x9, x2 + mul x10, x10, x2 + mul x11, x11, x2 + adds x10, x10, x22 + mul x12, x12, x2 + adcs x11, x11, x23 + mul x13, x13, x2 + adcs x12, x12, x24 + mul x14, x14, x2 + adcs x13, x13, x25 + adcs x14, x14 ,x26 + adc x28, x28, xzr + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x27, x28 + + extr x3, x4, x3, #62 + extr x4, x5, x4, #62 + extr x5, x6, x5, #62 + asr x28, x9, #63 + extr x6, x7, x6, #62 + extr x7, x8, x7, #62 + extr x8, x9, x8, #62 + + eor x3, x3, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + adcs x6, x6, xzr + eor x8, x8, x28 + stp x3, x4, [x0,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x0,#8*2] + adc x8, x8, xzr + stp x7, x8, [x0,#8*4] + + eor x15, x15, x28 + eor x16, x16, x28 + sub x15, x15, x28 + sub x16, x16, x28 + + ret + +.def __ab_approximation_62; +.type 32; +.endef +.p2align 4 +__ab_approximation_62: + ldp x7, x8, [x1,#8*4] + ldp x13, x14, [x1,#8*10] + ldp x5, x6, [x1,#8*2] + ldp x11, x12, [x1,#8*8] + +.Lab_approximation_62_loaded: + orr x22, x8, x14 // check top-most limbs, ... + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x22, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + ldp x3, x4, [x1,#8*0] + ldp x9, x10, [x1,#8*6] + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x22, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x22, x8, x14 + csel x13, x13, x10, ne + + clz x22, x22 + cmp x22, #64 + csel x22, x22, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x23, x22 + + lslv x8, x8, x22 // align high limbs to the left + lslv x14, x14, x22 + lsrv x7, x7, x23 + lsrv x13, x13, x23 + and x7, x7, x23, asr#6 + and x13, x13, x23, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + b __inner_loop_62 + ret + +.def __inner_loop_62; +.type 32; +.endef +.p2align 4 +__inner_loop_62: + mov x15, #1 // |f0|=1 + mov x16, #0 // |g0|=0 + mov x17, #0 // |f1|=0 + mov x19, #1 // |g1|=1 + +.Loop_62: + sbfx x28, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + subs x24, x9, x3 // |b_|-|a_| + and x22, x9, x28 + sbc x25, x14, x8 + and x23, x14, x28 + subs x26, x3, x22 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x22, x15 + sbcs x27, x8, x23 + mov x23, x16 + csel x9, x9, x3, hs // |b_| = |a_| + csel x14, x14, x8, hs + csel x3, x26, x24, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x8, x27, x25, hs + csel x15, x15, x17, hs // exchange |f0| and |f1| + csel x17, x17, x22, hs + csel x16, x16, x19, hs // exchange |g0| and |g1| + csel x19, x19, x23, hs + extr x3, x8, x3, #1 + lsr x8, x8, #1 + and x22, x17, x28 + and x23, x19, x28 + add x17, x17, x17 // |f1|<<=1 + add x19, x19, x19 // |g1|<<=1 + sub x15, x15, x22 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x16, x16, x23 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62 + + ret + diff --git a/crypto/blst_src/build/coff/ct_is_square_mod_384-armv8.S b/crypto/blst_src/build/coff/ct_is_square_mod_384-armv8.S new file mode 100644 index 00000000000..34336ff486b --- /dev/null +++ b/crypto/blst_src/build/coff/ct_is_square_mod_384-armv8.S @@ -0,0 +1,334 @@ +.text + +.globl ct_is_square_mod_384 +.def ct_is_square_mod_384; +.type 32; +.endef +.p2align 5 +ct_is_square_mod_384: +.long 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #512 + + ldp x3, x4, [x0,#8*0] // load input + ldp x5, x6, [x0,#8*2] + ldp x7, x8, [x0,#8*4] + + add x0, sp, #255 // find closest 256-byte-aligned spot + and x0, x0, #-256 // in the frame... + + ldp x9, x10, [x1,#8*0] // load modulus + ldp x11, x12, [x1,#8*2] + ldp x13, x14, [x1,#8*4] + + stp x3, x4, [x0,#8*6] // copy input to |a| + stp x5, x6, [x0,#8*8] + stp x7, x8, [x0,#8*10] + stp x9, x10, [x0,#8*0] // copy modulus to |b| + stp x11, x12, [x0,#8*2] + stp x13, x14, [x0,#8*4] + + eor x2, x2, x2 // init the .Legendre symbol + mov x15, #24 // 24 is 768/30-1 + b .Loop_is_square + +.p2align 4 +.Loop_is_square: + bl __ab_approximation_30 + sub x15, x15, #1 + + eor x1, x0, #128 // pointer to dst |b| + bl __smul_384_n_shift_by_30 + + mov x19, x16 // |f0| + mov x20, x17 // |g0| + add x1, x1, #8*6 // pointer to dst |a| + bl __smul_384_n_shift_by_30 + + ldp x9, x10, [x1,#-8*6] + eor x0, x0, #128 // flip-flop src |a|b| + and x27, x27, x9 // if |a| was negative, + add x2, x2, x27, lsr#1 // adjust |L| + + cbnz x15, .Loop_is_square + + ////////////////////////////////////////// last iteration + //bl __ab_approximation_30 // |a| and |b| are exact, + //ldr x8, [x0,#8*6] // and loaded + //ldr x14, [x0,#8*0] + mov x15, #48 // 48 is 768%30 + 30 + bl __inner_loop_48 + ldr x30, [x29,#8] + + and x0, x2, #1 + eor x0, x0, #1 + + add sp, sp, #512 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 +.long 3573752767 + ret + + +.def __smul_384_n_shift_by_30; +.type 32; +.endef +.p2align 5 +__smul_384_n_shift_by_30: + ldp x3, x4, [x0,#8*0+0] // load |b| (or |a|) + asr x27, x20, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x5, x6, [x0,#8*2+0] + eor x20, x20, x27 // conditionally negate |g1| (or |f1|) + ldp x7, x8, [x0,#8*4+0] + + eor x3, x3, x27 // conditionally negate |b| (or |a|) + sub x20, x20, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + umulh x21, x3, x20 + adcs x6, x6, xzr + umulh x22, x4, x20 + eor x8, x8, x27 + umulh x23, x5, x20 + adcs x7, x7, xzr + umulh x24, x6, x20 + adc x8, x8, xzr + + umulh x25, x7, x20 + and x28, x20, x27 + umulh x26, x8, x20 + neg x28, x28 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x21 + mul x6, x6, x20 + adcs x5, x5, x22 + mul x7, x7, x20 + adcs x6, x6, x23 + mul x8, x8, x20 + adcs x7, x7, x24 + adcs x8, x8 ,x25 + adc x26, x26, x28 + ldp x9, x10, [x0,#8*0+48] // load |b| (or |a|) + asr x27, x19, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x11, x12, [x0,#8*2+48] + eor x19, x19, x27 // conditionally negate |g1| (or |f1|) + ldp x13, x14, [x0,#8*4+48] + + eor x9, x9, x27 // conditionally negate |b| (or |a|) + sub x19, x19, x27 + eor x10, x10, x27 + adds x9, x9, x27, lsr#63 + eor x11, x11, x27 + adcs x10, x10, xzr + eor x12, x12, x27 + adcs x11, x11, xzr + eor x13, x13, x27 + umulh x21, x9, x19 + adcs x12, x12, xzr + umulh x22, x10, x19 + eor x14, x14, x27 + umulh x23, x11, x19 + adcs x13, x13, xzr + umulh x24, x12, x19 + adc x14, x14, xzr + + umulh x25, x13, x19 + and x28, x19, x27 + umulh x27, x14, x19 + neg x28, x28 + mul x9, x9, x19 + mul x10, x10, x19 + mul x11, x11, x19 + adds x10, x10, x21 + mul x12, x12, x19 + adcs x11, x11, x22 + mul x13, x13, x19 + adcs x12, x12, x23 + mul x14, x14, x19 + adcs x13, x13, x24 + adcs x14, x14 ,x25 + adc x27, x27, x28 + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x26, x27 + + extr x3, x4, x3, #30 + extr x4, x5, x4, #30 + extr x5, x6, x5, #30 + asr x27, x9, #63 + extr x6, x7, x6, #30 + extr x7, x8, x7, #30 + extr x8, x9, x8, #30 + + eor x3, x3, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + adcs x6, x6, xzr + eor x8, x8, x27 + stp x3, x4, [x1,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x1,#8*2] + adc x8, x8, xzr + stp x7, x8, [x1,#8*4] + + ret + +.def __ab_approximation_30; +.type 32; +.endef +.p2align 4 +__ab_approximation_30: + ldp x13, x14, [x0,#8*4] // |a| is still in registers + ldp x11, x12, [x0,#8*2] + + orr x21, x8, x14 // check top-most limbs, ... + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x21, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x21, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x21, x8, x14 // and one more, ... + csel x13, x13, x10, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x3, ne + orr x21, x8, x14 + csel x13, x13, x9, ne + + clz x21, x21 + cmp x21, #64 + csel x21, x21, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x22, x21 + + lslv x8, x8, x21 // align high limbs to the left + lslv x14, x14, x21 + lsrv x7, x7, x22 + lsrv x13, x13, x22 + and x7, x7, x22, asr#6 + and x13, x13, x22, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + bfxil x8, x3, #0, #32 + bfxil x14, x9, #0, #32 + + b __inner_loop_30 + ret + + +.def __inner_loop_30; +.type 32; +.endef +.p2align 4 +__inner_loop_30: + mov x28, #30 + mov x17, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x20, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x27,#0x7FFFFFFF7FFFFFFF + +.Loop_30: + sbfx x24, x8, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x8, x14 + sub x28, x28, #1 + and x21, x14, x24 + + sub x22, x14, x8 // |b_|-|a_| + subs x23, x8, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 // L + (a_ & b_) >> 1 + mov x21, x20 + csel x14, x14, x8, hs // |b_| = |a_| + csel x8, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x20, x20, x17, hs // exchange |fg0| and |fg1| + csel x17, x17, x21, hs + csel x2, x2, x25, hs + lsr x8, x8, #1 + and x21, x20, x24 + and x22, x27, x24 + add x23, x14, #2 + sub x17, x17, x21 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x20, x20, x20 // |f1|<<=1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + add x17, x17, x22 + sub x20, x20, x27 + + cbnz x28, .Loop_30 + + mov x27, #0x7FFFFFFF + ubfx x16, x17, #0, #32 + ubfx x17, x17, #32, #32 + ubfx x19, x20, #0, #32 + ubfx x20, x20, #32, #32 + sub x16, x16, x27 // remove the bias + sub x17, x17, x27 + sub x19, x19, x27 + sub x20, x20, x27 + + ret + +.def __inner_loop_48; +.type 32; +.endef +.p2align 4 +__inner_loop_48: +.Loop_48: + sbfx x24, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x3, x9 + sub x15, x15, #1 + and x21, x9, x24 + sub x22, x9, x3 // |b_|-|a_| + subs x23, x3, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 + csel x9, x9, x3, hs // |b_| = |a_| + csel x3, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x2, x2, x25, hs + add x23, x9, #2 + lsr x3, x3, #1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + + cbnz x15, .Loop_48 + + ret + diff --git a/crypto/blst_src/build/coff/ct_is_square_mod_384-x86_64.s b/crypto/blst_src/build/coff/ct_is_square_mod_384-x86_64.s new file mode 100644 index 00000000000..ee4790321e6 --- /dev/null +++ b/crypto/blst_src/build/coff/ct_is_square_mod_384-x86_64.s @@ -0,0 +1,505 @@ +.text + +.globl ct_is_square_mod_384 +.def ct_is_square_mod_384; .scl 2; .type 32; .endef +.p2align 5 +ct_is_square_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_ct_is_square_mod_384: + movq %rcx,%rdi + movq %rdx,%rsi + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $536,%rsp + +.LSEH_body_ct_is_square_mod_384: + + + leaq 24+255(%rsp),%rax + andq $-256,%rax + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbx + movq 24(%rsi),%rcx + movq 32(%rsi),%rdx + movq 40(%rsi),%rdi + movq %rax,%rsi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rcx,72(%rax) + movq %rdx,80(%rax) + movq %rdi,88(%rax) + + xorq %rbp,%rbp + movl $24,%ecx + jmp .Loop_is_square + +.p2align 5 +.Loop_is_square: + movl %ecx,16(%rsp) + + call __ab_approximation_30 + movq %rax,0(%rsp) + movq %rbx,8(%rsp) + + movq $128+48,%rdi + xorq %rsi,%rdi + call __smulq_384_n_shift_by_30 + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq -48(%rdi),%rdi + call __smulq_384_n_shift_by_30 + + movl 16(%rsp),%ecx + xorq $128,%rsi + + andq 48(%rdi),%r14 + shrq $1,%r14 + addq %r14,%rbp + + subl $1,%ecx + jnz .Loop_is_square + + + + + movq 48(%rsi),%r9 + call __inner_loop_48 + + movq $1,%rax + andq %rbp,%rax + xorq $1,%rax + + leaq 536(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_ct_is_square_mod_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_ct_is_square_mod_384: + +.def __smulq_384_n_shift_by_30; .scl 3; .type 32; .endef +.p2align 5 +__smulq_384_n_shift_by_30: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r14 + andq %rbx,%r14 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r14 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r14 + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r15 + andq %rbx,%r15 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r15 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r15 + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %r15,%r14 + + shrdq $30,%r9,%r8 + shrdq $30,%r10,%r9 + shrdq $30,%r11,%r10 + shrdq $30,%r12,%r11 + shrdq $30,%r13,%r12 + shrdq $30,%r14,%r13 + + sarq $63,%r14 + xorq %rbx,%rbx + subq %r14,%rbx + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + +.def __ab_approximation_30; .scl 3; .type 32; .endef +.p2align 5 +__ab_approximation_30: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 88(%rsi),%rbx + movq 80(%rsi),%r15 + movq 72(%rsi),%r14 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r11,%r12 + movq 64(%rsi),%r11 + cmovzq %r14,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r10,%r12 + movq 56(%rsi),%r10 + cmovzq %r11,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r9,%r12 + movq 48(%rsi),%r9 + cmovzq %r10,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r8,%r12 + cmovzq %r9,%r15 + + movq %r13,%rax + orq %rbx,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r13 + cmovzq %r9,%rbx + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%r12,%r13 + shldq %cl,%r15,%rbx + + movq $0xFFFFFFFF00000000,%rax + movl %r8d,%r8d + movl %r9d,%r9d + andq %rax,%r13 + andq %rax,%rbx + orq %r13,%r8 + orq %rbx,%r9 + + jmp __inner_loop_30 + + .byte 0xf3,0xc3 + +.def __inner_loop_30; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_30: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rbx + movq $0x800000007FFFFFFF,%rcx + leaq -1(%rbx),%r15 + movl $30,%edi + +.Loop_30: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbx,%r12 + movq %rcx,%r13 + movq %rbp,%r14 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rcx,%rbx + cmovbq %r12,%rcx + cmovbq %rax,%rbp + + subq %r9,%r8 + subq %rcx,%rbx + addq %r15,%rbx + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbx + cmovzq %r13,%rcx + cmovzq %r14,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rcx,%rcx + leaq (%rax,%rbp,1),%rbp + subq %r15,%rcx + + subl $1,%edi + jnz .Loop_30 + + shrq $32,%r15 + movl %ebx,%eax + shrq $32,%rbx + movl %ecx,%edx + shrq $32,%rcx + subq %r15,%rax + subq %r15,%rbx + subq %r15,%rdx + subq %r15,%rcx + + .byte 0xf3,0xc3 + + +.def __inner_loop_48; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_48: + .byte 0xf3,0x0f,0x1e,0xfa + + movl $48,%edi + +.Loop_48: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbp,%r12 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rax,%rbp + + subq %r9,%r8 + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rax,%rbp + + subl $1,%edi + jnz .Loop_48 + + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_ct_is_square_mod_384 +.rva .LSEH_body_ct_is_square_mod_384 +.rva .LSEH_info_ct_is_square_mod_384_prologue + +.rva .LSEH_body_ct_is_square_mod_384 +.rva .LSEH_epilogue_ct_is_square_mod_384 +.rva .LSEH_info_ct_is_square_mod_384_body + +.rva .LSEH_epilogue_ct_is_square_mod_384 +.rva .LSEH_end_ct_is_square_mod_384 +.rva .LSEH_info_ct_is_square_mod_384_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_ct_is_square_mod_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_ct_is_square_mod_384_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x43,0x00 +.byte 0x00,0xe4,0x44,0x00 +.byte 0x00,0xd4,0x45,0x00 +.byte 0x00,0xc4,0x46,0x00 +.byte 0x00,0x34,0x47,0x00 +.byte 0x00,0x54,0x48,0x00 +.byte 0x00,0x74,0x4a,0x00 +.byte 0x00,0x64,0x4b,0x00 +.byte 0x00,0x01,0x49,0x00 +.LSEH_info_ct_is_square_mod_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s b/crypto/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..42f058a3c8d --- /dev/null +++ b/crypto/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s @@ -0,0 +1,1221 @@ +.text + +.globl ct_inverse_mod_383 +.def ct_inverse_mod_383; .scl 2; .type 32; .endef +.p2align 5 +ct_inverse_mod_383: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_ct_inverse_mod_383: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $1112,%rsp + +.LSEH_body_ct_inverse_mod_383: + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + xorq $256+96,%rsi + movl $62,%edi + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 48(%rsi),%r10 + movq 56(%rsi),%r11 + call __inner_loop_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + movq %r8,0(%rdi) + movq %r10,48(%rdi) + + + + leaq 96(%rsi),%rsi + leaq 96(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + + xorq $256+96,%rsi + movl $22,%edi + + movq 0(%rsi),%r8 + xorq %r9,%r9 + movq 48(%rsi),%r10 + xorq %r11,%r11 + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_ct_inverse_mod_383: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_ct_inverse_mod_383: +.def __smulq_767x63; .scl 3; .type 32; .endef +.p2align 5 +__smulq_767x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + movq %r9,8(%rdi) + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + movq %r10,16(%rdi) + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %r11,24(%rdi) + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + movq %r12,32(%rdi) + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + movq %r13,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + movq %rdx,%rsi + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rsi + addq %rax,%rsi + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rbx + xorq %rdx,%rbp + xorq %rdx,%rcx + xorq %rdx,%rdi + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulq %rsi + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rsi + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rsi + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rsi + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rsi + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rsi + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %rdx,%rbx + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + mulq %rsi + addq %rax,%rbp + movq %rcx,%rax + adcq $0,%rdx + movq %rdx,%rcx + mulq %rsi + addq %rax,%rcx + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%rdi + movq 8(%rsp),%rdx + imulq %rsi,%rax + movq 16(%rsp),%rsi + addq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 + +.def __smulq_383x63; .scl 3; .type 32; .endef +.p2align 5 +__smulq_383x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + +.def __smulq_383_n_shift_by_62; .scl 3; .type 32; .endef +.p2align 5 +__smulq_383_n_shift_by_62: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq 48(%rsi),%rsi + movq %rdx,%r14 + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $62,%r9,%r8 + shrdq $62,%r10,%r9 + shrdq $62,%r11,%r10 + shrdq $62,%r12,%r11 + shrdq $62,%r13,%r12 + shrdq $62,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 + +.def __ab_approximation_62; .scl 3; .type 32; .endef +.p2align 5 +__ab_approximation_62: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 16(%rsi),%r8 + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 8(%rsi),%r8 + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 0(%rsi),%r8 + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + jmp __inner_loop_62 + + .byte 0xf3,0xc3 + +.def __inner_loop_62; .scl 3; .type 32; .endef +.p2align 3 +.long 0 +__inner_loop_62: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + movq %rsi,8(%rsp) + +.Loop_62: + xorq %rax,%rax + xorq %rbx,%rbx + testq $1,%r8 + movq %r10,%rbp + movq %r11,%r14 + cmovnzq %r10,%rax + cmovnzq %r11,%rbx + subq %r8,%rbp + sbbq %r9,%r14 + movq %r8,%r15 + movq %r9,%rsi + subq %rax,%r8 + sbbq %rbx,%r9 + cmovcq %rbp,%r8 + cmovcq %r14,%r9 + cmovcq %r15,%r10 + cmovcq %rsi,%r11 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrdq $1,%r9,%r8 + shrq $1,%r9 + testq $1,%r15 + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz .Loop_62 + + movq 8(%rsp),%rsi + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_ct_inverse_mod_383 +.rva .LSEH_body_ct_inverse_mod_383 +.rva .LSEH_info_ct_inverse_mod_383_prologue + +.rva .LSEH_body_ct_inverse_mod_383 +.rva .LSEH_epilogue_ct_inverse_mod_383 +.rva .LSEH_info_ct_inverse_mod_383_body + +.rva .LSEH_epilogue_ct_inverse_mod_383 +.rva .LSEH_end_ct_inverse_mod_383 +.rva .LSEH_info_ct_inverse_mod_383_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_ct_inverse_mod_383_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_ct_inverse_mod_383_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x8b,0x00 +.byte 0x00,0xe4,0x8c,0x00 +.byte 0x00,0xd4,0x8d,0x00 +.byte 0x00,0xc4,0x8e,0x00 +.byte 0x00,0x34,0x8f,0x00 +.byte 0x00,0x54,0x90,0x00 +.byte 0x00,0x74,0x92,0x00 +.byte 0x00,0x64,0x93,0x00 +.byte 0x00,0x01,0x91,0x00 +.LSEH_info_ct_inverse_mod_383_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s b/crypto/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..7c13e56eb2a --- /dev/null +++ b/crypto/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s @@ -0,0 +1,1596 @@ +.text + +.globl ctx_inverse_mod_383 +.def ctx_inverse_mod_383; .scl 2; .type 32; .endef +.p2align 5 +ctx_inverse_mod_383: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_ctx_inverse_mod_383: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $1112,%rsp + +.LSEH_body_ctx_inverse_mod_383: + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + + xorq $256+96,%rsi + movl $53,%edi + + movq 0(%rsi),%r8 + + movq 48(%rsi),%r10 + + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulx_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_ctx_inverse_mod_383: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_ctx_inverse_mod_383: +.def __smulx_767x63; .scl 3; .type 32; .endef +.p2align 5 +__smulx_767x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + movq %rcx,%rax + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + sarq $63,%rax + xorq %rsi,%rsi + subq %rax,%rsi + + xorq %rax,%rdx + addq %rsi,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %rax,%r13 + xorq %rax,%r14 + xorq %rax,%r15 + xorq %rax,%rbx + xorq %rax,%rbp + xorq %rax,%rcx + xorq %rax,%rdi + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulxq %r8,%r8,%rax + mulxq %r9,%r9,%rsi + addq %rax,%r9 + mulxq %r10,%r10,%rax + adcq %rsi,%r10 + mulxq %r11,%r11,%rsi + adcq %rax,%r11 + mulxq %r12,%r12,%rax + adcq %rsi,%r12 + mulxq %r13,%r13,%rsi + adcq %rax,%r13 + mulxq %r14,%r14,%rax + adcq %rsi,%r14 + mulxq %r15,%r15,%rsi + adcq %rax,%r15 + mulxq %rbx,%rbx,%rax + adcq %rsi,%rbx + mulxq %rbp,%rbp,%rsi + adcq %rax,%rbp + mulxq %rcx,%rcx,%rax + adcq %rsi,%rcx + mulxq %rdi,%rdi,%rsi + movq 8(%rsp),%rdx + movq 16(%rsp),%rsi + adcq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 + +.def __smulx_383x63; .scl 3; .type 32; .endef +.p2align 5 +__smulx_383x63: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + movq %rcx,%rdx + adcq %rbp,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + +.def __smulx_383_n_shift_by_31; .scl 3; .type 32; .endef +.p2align 5 +__smulx_383_n_shift_by_31: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + xorq %r14,%r14 + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq %rdx,%r14 + + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%rax + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%r12,%r11 + shrdq $31,%rax,%r12 + shrdq $31,%r14,%rax + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 + +.def __smulx_191_n_shift_by_31; .scl 3; .type 32; .endef +.p2align 5 +__smulx_191_n_shift_by_31: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %r10,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r10 + addq %rbp,%r9 + adcq $0,%r10 + imulq %rdx + addq %rax,%r10 + adcq $0,%rdx + movq %rdx,%r14 + movq %rcx,%rdx + movq 48+0(%rsi),%r11 + movq 48+8(%rsi),%r12 + movq 48+16(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r11,%r11,%rbp + mulxq %r12,%r12,%r13 + addq %rbp,%r12 + adcq $0,%r13 + imulq %rdx + addq %rax,%r13 + adcq $0,%rdx + addq %r8,%r11 + adcq %r9,%r12 + adcq %r10,%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r12,%r11 + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r11,0(%rdi) + movq %r12,8(%rdi) + movq %r13,16(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 + +.def __ab_approximation_31; .scl 3; .type 32; .endef +.p2align 5 +__ab_approximation_31: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 16(%rsi),%r8 + cmovzq %r10,%rbp + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 8(%rsi),%r8 + cmovzq %r10,%rbp + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + andnq %r9,%rax,%r9 + andnq %r11,%rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31 + + .byte 0xf3,0xc3 + +.def __inner_loop_31; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_31: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +.Loop_31: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edi + jnz .Loop_31 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 + + +.def __inner_loop_62; .scl 3; .type 32; .endef +.p2align 5 +__inner_loop_62: + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + +.Loop_62: + xorq %rax,%rax + testq $1,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq $1,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz .Loop_62 + + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_ctx_inverse_mod_383 +.rva .LSEH_body_ctx_inverse_mod_383 +.rva .LSEH_info_ctx_inverse_mod_383_prologue + +.rva .LSEH_body_ctx_inverse_mod_383 +.rva .LSEH_epilogue_ctx_inverse_mod_383 +.rva .LSEH_info_ctx_inverse_mod_383_body + +.rva .LSEH_epilogue_ctx_inverse_mod_383 +.rva .LSEH_end_ctx_inverse_mod_383 +.rva .LSEH_info_ctx_inverse_mod_383_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_ctx_inverse_mod_383_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_ctx_inverse_mod_383_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x8b,0x00 +.byte 0x00,0xe4,0x8c,0x00 +.byte 0x00,0xd4,0x8d,0x00 +.byte 0x00,0xc4,0x8e,0x00 +.byte 0x00,0x34,0x8f,0x00 +.byte 0x00,0x54,0x90,0x00 +.byte 0x00,0x74,0x92,0x00 +.byte 0x00,0x64,0x93,0x00 +.byte 0x00,0x01,0x91,0x00 +.LSEH_info_ctx_inverse_mod_383_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/div3w-armv8.S b/crypto/blst_src/build/coff/div3w-armv8.S new file mode 100644 index 00000000000..c17b9e38336 --- /dev/null +++ b/crypto/blst_src/build/coff/div3w-armv8.S @@ -0,0 +1,94 @@ +.text + +.globl div_3_limbs +.def div_3_limbs; +.type 32; +.endef +.p2align 5 +div_3_limbs: + ldp x4,x5,[x0] // load R + eor x0,x0,x0 // Q = 0 + mov x3,#64 // loop counter + nop + +.Loop: + subs x6,x4,x1 // R - D + add x0,x0,x0 // Q <<= 1 + sbcs x7,x5,x2 + add x0,x0,#1 // Q + speculative bit + csel x4,x4,x6,lo // select between R and R - D + extr x1,x2,x1,#1 // D >>= 1 + csel x5,x5,x7,lo + lsr x2,x2,#1 + sbc x0,x0,xzr // subtract speculative bit + sub x3,x3,#1 + cbnz x3,.Loop + + asr x3,x0,#63 // top bit -> mask + add x0,x0,x0 // Q <<= 1 + subs x6,x4,x1 // R - D + add x0,x0,#1 // Q + specilative bit + sbcs x7,x5,x2 + sbc x0,x0,xzr // subtract speculative bit + + orr x0,x0,x3 // all ones if overflow + + ret + +.globl quot_rem_128 +.def quot_rem_128; +.type 32; +.endef +.p2align 5 +quot_rem_128: + ldp x3,x4,[x1] + + mul x5,x3,x2 // divisor[0:1} * quotient + umulh x6,x3,x2 + mul x11, x4,x2 + umulh x7,x4,x2 + + ldp x8,x9,[x0] // load 3 limbs of the dividend + ldr x10,[x0,#16] + + adds x6,x6,x11 + adc x7,x7,xzr + + subs x8,x8,x5 // dividend - divisor * quotient + sbcs x9,x9,x6 + sbcs x10,x10,x7 + sbc x5,xzr,xzr // borrow -> mask + + add x2,x2,x5 // if borrowed, adjust the quotient ... + and x3,x3,x5 + and x4,x4,x5 + adds x8,x8,x3 // ... and add divisor + adc x9,x9,x4 + + stp x8,x9,[x0] // save 2 limbs of the remainder + str x2,[x0,#16] // and one limb of the quotient + + mov x0,x2 // return adjusted quotient + + ret + + +.globl quot_rem_64 +.def quot_rem_64; +.type 32; +.endef +.p2align 5 +quot_rem_64: + ldr x3,[x1] + ldr x8,[x0] // load 1 limb of the dividend + + mul x5,x3,x2 // divisor * quotient + + sub x8,x8,x5 // dividend - divisor * quotient + + stp x8,x2,[x0] // save remainder and quotient + + mov x0,x2 // return quotient + + ret + diff --git a/crypto/blst_src/build/coff/div3w-x86_64.s b/crypto/blst_src/build/coff/div3w-x86_64.s new file mode 100644 index 00000000000..fcfe54480be --- /dev/null +++ b/crypto/blst_src/build/coff/div3w-x86_64.s @@ -0,0 +1,140 @@ +.text + +.globl div_3_limbs + +.def div_3_limbs; .scl 2; .type 32; .endef +.p2align 5 +div_3_limbs: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%rax +.LSEH_begin_div_3_limbs: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + movq (%rdi),%r8 + movq 8(%rdi),%r9 + xorq %rax,%rax + movl $64,%ecx + +.Loop: + movq %r8,%r10 + subq %rsi,%r8 + movq %r9,%r11 + sbbq %rdx,%r9 + leaq 1(%rax,%rax,1),%rax + movq %rdx,%rdi + cmovcq %r10,%r8 + cmovcq %r11,%r9 + sbbq $0,%rax + shlq $63,%rdi + shrq $1,%rsi + shrq $1,%rdx + orq %rdi,%rsi + subl $1,%ecx + jnz .Loop + + leaq 1(%rax,%rax,1),%rcx + sarq $63,%rax + + subq %rsi,%r8 + sbbq %rdx,%r9 + sbbq $0,%rcx + + orq %rcx,%rax + + movq 8(%rsp),%rdi + movq 16(%rsp),%rsi + .byte 0xf3,0xc3 +.LSEH_end_div_3_limbs: +.globl quot_rem_128 + +.def quot_rem_128; .scl 2; .type 32; .endef +.p2align 5 +quot_rem_128: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%rax +.LSEH_begin_quot_rem_128: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + movq %rdx,%rax + movq %rdx,%rcx + + mulq 0(%rsi) + movq %rax,%r8 + movq %rcx,%rax + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r9 + adcq $0,%rdx + + movq 0(%rdi),%r10 + movq 8(%rdi),%r11 + movq 16(%rdi),%rax + + subq %r8,%r10 + sbbq %r9,%r11 + sbbq %rdx,%rax + sbbq %r8,%r8 + + addq %r8,%rcx + movq %r8,%r9 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + addq %r8,%r10 + adcq %r9,%r11 + + movq %r10,0(%rdi) + movq %r11,8(%rdi) + movq %rcx,16(%rdi) + + movq %rcx,%rax + + movq 8(%rsp),%rdi + movq 16(%rsp),%rsi + .byte 0xf3,0xc3 +.LSEH_end_quot_rem_128: + + + + + +.globl quot_rem_64 + +.def quot_rem_64; .scl 2; .type 32; .endef +.p2align 5 +quot_rem_64: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%rax +.LSEH_begin_quot_rem_64: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + movq %rdx,%rax + imulq 0(%rsi),%rdx + + movq 0(%rdi),%r10 + + subq %rdx,%r10 + + movq %r10,0(%rdi) + movq %rax,8(%rdi) + + movq 8(%rsp),%rdi + movq 16(%rsp),%rsi + .byte 0xf3,0xc3 +.LSEH_end_quot_rem_64: +.section .pdata +.p2align 2 +.section .xdata +.p2align 3 diff --git a/crypto/blst_src/build/coff/mul_mont_256-armv8.S b/crypto/blst_src/build/coff/mul_mont_256-armv8.S new file mode 100644 index 00000000000..8cadbb89344 --- /dev/null +++ b/crypto/blst_src/build/coff/mul_mont_256-armv8.S @@ -0,0 +1,474 @@ +.text + +.globl mul_mont_sparse_256 + +.def mul_mont_sparse_256; +.type 32; +.endef +.p2align 5 +mul_mont_sparse_256: + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x10,x11,[x1] + ldr x9, [x2] + ldp x12,x13,[x1,#16] + + mul x19,x10,x9 + ldp x5,x6,[x3] + mul x20,x11,x9 + ldp x7,x8,[x3,#16] + mul x21,x12,x9 + mul x22,x13,x9 + + umulh x14,x10,x9 + umulh x15,x11,x9 + mul x3,x4,x19 + umulh x16,x12,x9 + umulh x17,x13,x9 + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,xzr, x17 + mul x17,x8,x3 + ldr x9,[x2,8*1] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*2] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*3] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + adcs x20,x21,x15 + adcs x21,x22,x16 + adcs x22,x23,x17 + adc x23,xzr,xzr + + subs x14,x19,x5 + sbcs x15,x20,x6 + sbcs x16,x21,x7 + sbcs x17,x22,x8 + sbcs xzr, x23,xzr + + csel x19,x19,x14,lo + csel x20,x20,x15,lo + csel x21,x21,x16,lo + csel x22,x22,x17,lo + + stp x19,x20,[x0] + stp x21,x22,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + ret + +.globl sqr_mont_sparse_256 + +.def sqr_mont_sparse_256; +.type 32; +.endef +.p2align 5 +sqr_mont_sparse_256: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + mov x4,x3 + + //////////////////////////////////////////////////////////////// + // | | | | | |a1*a0| | + // | | | | |a2*a0| | | + // | |a3*a2|a3*a0| | | | + // | | | |a2*a1| | | | + // | | |a3*a1| | | | | + // *| | | | | | | | 2| + // +|a3*a3|a2*a2|a1*a1|a0*a0| + // |--+--+--+--+--+--+--+--| + // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is x10 + // + // "can't overflow" below mark carrying into high part of + // multiplication result, which can't overflow, because it + // can never be all ones. + + mul x11,x6,x5 // a[1]*a[0] + umulh x15,x6,x5 + mul x12,x7,x5 // a[2]*a[0] + umulh x16,x7,x5 + mul x13,x8,x5 // a[3]*a[0] + umulh x19,x8,x5 + + adds x12,x12,x15 // accumulate high parts of multiplication + mul x14,x7,x6 // a[2]*a[1] + umulh x15,x7,x6 + adcs x13,x13,x16 + mul x16,x8,x6 // a[3]*a[1] + umulh x17,x8,x6 + adc x19,x19,xzr // can't overflow + + mul x20,x8,x7 // a[3]*a[2] + umulh x21,x8,x7 + + adds x15,x15,x16 // accumulate high parts of multiplication + mul x10,x5,x5 // a[0]*a[0] + adc x16,x17,xzr // can't overflow + + adds x13,x13,x14 // accumulate low parts of multiplication + umulh x5,x5,x5 + adcs x19,x19,x15 + mul x15,x6,x6 // a[1]*a[1] + adcs x20,x20,x16 + umulh x6,x6,x6 + adc x21,x21,xzr // can't overflow + + adds x11,x11,x11 // acc[1-6]*=2 + mul x16,x7,x7 // a[2]*a[2] + adcs x12,x12,x12 + umulh x7,x7,x7 + adcs x13,x13,x13 + mul x17,x8,x8 // a[3]*a[3] + adcs x19,x19,x19 + umulh x8,x8,x8 + adcs x20,x20,x20 + adcs x21,x21,x21 + adc x22,xzr,xzr + + adds x11,x11,x5 // +a[i]*a[i] + adcs x12,x12,x15 + adcs x13,x13,x6 + adcs x19,x19,x16 + adcs x20,x20,x7 + adcs x21,x21,x17 + adc x22,x22,x8 + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + adds x10,x10,x19 // accumulate upper half + adcs x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adc x19,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x19,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + +.globl from_mont_256 + +.def from_mont_256; +.type 32; +.endef +.p2align 5 +from_mont_256: +.long 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 +.long 3573752767 + ret + + +.globl redc_mont_256 + +.def redc_mont_256; +.type 32; +.endef +.p2align 5 +redc_mont_256: +.long 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + ldp x14,x15,[x1,#32] + ldp x16,x17,[x1,#48] + + adds x10,x10,x14 + adcs x11,x11,x15 + adcs x12,x12,x16 + adcs x13,x13,x17 + adc x9,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x9,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 +.long 3573752767 + ret + + +.def __mul_by_1_mont_256; +.type 32; +.endef +.p2align 5 +__mul_by_1_mont_256: + mul x3,x4,x10 + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + adc x13,x9,x17 + + ret + diff --git a/crypto/blst_src/build/coff/mul_mont_384-armv8.S b/crypto/blst_src/build/coff/mul_mont_384-armv8.S new file mode 100644 index 00000000000..074f38c495c --- /dev/null +++ b/crypto/blst_src/build/coff/mul_mont_384-armv8.S @@ -0,0 +1,2424 @@ +.text + +.globl add_mod_384x384 +.def add_mod_384x384; +.type 32; +.endef +.p2align 5 +add_mod_384x384: +.long 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __add_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 +.long 3573752767 + ret + + +.def __add_mod_384x384; +.type 32; +.endef +.p2align 5 +__add_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + stp x11, x12, [x0] + adcs x15,x15,x23 + ldp x11, x12, [x1,#48] + adcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + adcs x11,x11,x19 + stp x15, x16, [x0,#32] + adcs x12,x12,x20 + ldp x15, x16, [x1,#80] + adcs x13,x13,x21 + ldp x23,x24,[x2,#80] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + stp x11,x12,[x0,#48] + csel x15,x15,x23,lo + stp x13,x14,[x0,#64] + csel x16,x16,x24,lo + stp x15,x16,[x0,#80] + + ret + + +.globl sub_mod_384x384 +.def sub_mod_384x384; +.type 32; +.endef +.p2align 5 +sub_mod_384x384: +.long 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 +.long 3573752767 + ret + + +.def __sub_mod_384x384; +.type 32; +.endef +.p2align 5 +__sub_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + stp x11, x12, [x0] + sbcs x15,x15,x23 + ldp x11, x12, [x1,#48] + sbcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + sbcs x11,x11,x19 + stp x15, x16, [x0,#32] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#80] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#80] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + ret + + +.def __add_mod_384; +.type 32; +.endef +.p2align 5 +__add_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + stp x11,x12,[x0] + csel x16,x16,x24,lo + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + +.def __sub_mod_384; +.type 32; +.endef +.p2align 5 +__sub_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0] + adc x16,x16,x24 + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + +.globl mul_mont_384x + +.def mul_mont_384x; +.type 32; +.endef +.p2align 5 +mul_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#288 // space for 3 768-bit vectors + + mov x26,x0 // save r_ptr + mov x27,x1 // save b_ptr + mov x28,x2 // save b_ptr + + sub x0,sp,#0 // mul_384(t0, a->re, b->re) + bl __mul_384 + + add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x2,x2,#48 + add x0,sp,#96 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + sub x2,x1,#48 + add x0,sp,#240 + bl __add_mod_384 + + add x1,x28,#0 + add x2,x28,#48 + add x0,sp,#192 // t2 + bl __add_mod_384 + + add x1,x0,#0 + add x2,x0,#48 + bl __mul_384 // mul_384(t2, a->re+a->im, b->re+b->im) + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,x0 + add x2,sp,#0 + bl __sub_mod_384x384 + + add x2,sp,#96 + bl __sub_mod_384x384 // t2 = t2-t0-t1 + + add x1,sp,#0 + add x2,sp,#96 + add x0,sp,#0 + bl __sub_mod_384x384 // t0 = t0-t1 + + add x1,sp,#0 // ret->re = redc(t0) + add x0,x26,#0 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + + add x1,sp,#192 // ret->im = redc(t2) + add x0,x0,#48 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#288 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl sqr_mont_384x + +.def sqr_mont_384x; +.type 32; +.endef +.p2align 5 +sqr_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 2 384-bit vectors + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + add x0,sp,#0 + bl __add_mod_384 // t0 = a->re + a->im + + add x0,sp,#48 + bl __sub_mod_384 // t1 = a->re - a->im + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __mul_mont_384 // mul_mont_384(ret->im, a->re, a->im) + + adds x11,x11,x11 // add with itself + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x19,x11,x19,lo + csel x20,x12,x20,lo + csel x21,x13,x21,lo + ldp x11,x12,[sp] + csel x22,x14,x22,lo + ldr x17, [sp,#48] + csel x23,x15,x23,lo + ldp x13,x14,[sp,#16] + csel x24,x16,x24,lo + ldp x15,x16,[sp,#32] + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + add x2,sp,#48 + bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl mul_mont_384 + +.def mul_mont_384; +.type 32; +.endef +.p2align 5 +mul_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.def __mul_mont_384; +.type 32; +.endef +.p2align 5 +__mul_mont_384: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + mov x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*1] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*2] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*3] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*4] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*5] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + adc x17,x17,xzr + + adds x19,x20,x26 + adcs x20,x21,x27 + adcs x21,x22,x28 + adcs x22,x23,x0 + adcs x23,x24,x1 + adcs x24,x25,x3 + adc x25,x17,xzr + + subs x26,x19,x5 + sbcs x27,x20,x6 + sbcs x28,x21,x7 + sbcs x0,x22,x8 + sbcs x1,x23,x9 + sbcs x3,x24,x10 + sbcs xzr, x25,xzr + + csel x11,x19,x26,lo + csel x12,x20,x27,lo + csel x13,x21,x28,lo + csel x14,x22,x0,lo + csel x15,x23,x1,lo + csel x16,x24,x3,lo + ret + + +.globl sqr_mont_384 + +.def sqr_mont_384; +.type 32; +.endef +.p2align 5 +sqr_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for 768-bit vector + mov x4,x3 // adjust for missing b_ptr + + mov x3,x0 // save r_ptr + mov x0,sp + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + mov x1,sp + mov x0,x3 // restore r_ptr + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl sqr_n_mul_mont_383 + +.def sqr_n_mul_mont_383; +.type 32; +.endef +.p2align 5 +sqr_n_mul_mont_383: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 768-bit vector + mov x17,x5 // save b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + mov x0,sp +.Loop_sqr_383: + bl __sqr_384 + sub x2,x2,#1 // counter + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,sp + bl __mul_by_1_mont_384 + + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // just accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + cbnz x2,.Loop_sqr_383 + + mov x2,x17 + ldr x17,[x17] + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + +.def __sqr_384; +.type 32; +.endef +.p2align 5 +__sqr_384: + mul x19,x12,x11 + mul x20,x13,x11 + mul x21,x14,x11 + mul x22,x15,x11 + mul x23,x16,x11 + + umulh x6,x12,x11 + umulh x7,x13,x11 + umulh x8,x14,x11 + umulh x9,x15,x11 + adds x20,x20,x6 + umulh x10,x16,x11 + adcs x21,x21,x7 + mul x7,x13,x12 + adcs x22,x22,x8 + mul x8,x14,x12 + adcs x23,x23,x9 + mul x9,x15,x12 + adc x24,xzr, x10 + mul x10,x16,x12 + + adds x21,x21,x7 + umulh x7,x13,x12 + adcs x22,x22,x8 + umulh x8,x14,x12 + adcs x23,x23,x9 + umulh x9,x15,x12 + adcs x24,x24,x10 + umulh x10,x16,x12 + adc x25,xzr,xzr + + mul x5,x11,x11 + adds x22,x22,x7 + umulh x11, x11,x11 + adcs x23,x23,x8 + mul x8,x14,x13 + adcs x24,x24,x9 + mul x9,x15,x13 + adc x25,x25,x10 + mul x10,x16,x13 + + adds x23,x23,x8 + umulh x8,x14,x13 + adcs x24,x24,x9 + umulh x9,x15,x13 + adcs x25,x25,x10 + umulh x10,x16,x13 + adc x26,xzr,xzr + + mul x6,x12,x12 + adds x24,x24,x8 + umulh x12, x12,x12 + adcs x25,x25,x9 + mul x9,x15,x14 + adc x26,x26,x10 + mul x10,x16,x14 + + adds x25,x25,x9 + umulh x9,x15,x14 + adcs x26,x26,x10 + umulh x10,x16,x14 + adc x27,xzr,xzr + mul x7,x13,x13 + adds x26,x26,x9 + umulh x13, x13,x13 + adc x27,x27,x10 + mul x8,x14,x14 + + mul x10,x16,x15 + umulh x14, x14,x14 + adds x27,x27,x10 + umulh x10,x16,x15 + mul x9,x15,x15 + adc x28,x10,xzr + + adds x19,x19,x19 + adcs x20,x20,x20 + adcs x21,x21,x21 + adcs x22,x22,x22 + adcs x23,x23,x23 + adcs x24,x24,x24 + adcs x25,x25,x25 + adcs x26,x26,x26 + umulh x15, x15,x15 + adcs x27,x27,x27 + mul x10,x16,x16 + adcs x28,x28,x28 + umulh x16, x16,x16 + adc x1,xzr,xzr + + adds x19,x19,x11 + adcs x20,x20,x6 + adcs x21,x21,x12 + adcs x22,x22,x7 + adcs x23,x23,x13 + adcs x24,x24,x8 + adcs x25,x25,x14 + stp x5,x19,[x0] + adcs x26,x26,x9 + stp x20,x21,[x0,#16] + adcs x27,x27,x15 + stp x22,x23,[x0,#32] + adcs x28,x28,x10 + stp x24,x25,[x0,#48] + adc x16,x16,x1 + stp x26,x27,[x0,#64] + stp x28,x16,[x0,#80] + + ret + +.globl sqr_384 + +.def sqr_384; +.type 32; +.endef +.p2align 5 +sqr_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl redc_mont_384 + +.def redc_mont_384; +.type 32; +.endef +.p2align 5 +redc_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl from_mont_384 + +.def from_mont_384; +.type 32; +.endef +.p2align 5 +from_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.def __mul_by_1_mont_384; +.type 32; +.endef +.p2align 5 +__mul_by_1_mont_384: + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + mul x26,x4,x11 + ldp x15,x16,[x1,#32] + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + ret + + +.def __redc_tail_mont_384; +.type 32; +.endef +.p2align 5 +__redc_tail_mont_384: + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + +.globl mul_384 + +.def mul_384; +.type 32; +.endef +.p2align 5 +mul_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + bl __mul_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.def __mul_384; +.type 32; +.endef +.p2align 5 +__mul_384: + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + + umulh x5,x11,x17 + umulh x6,x12,x17 + umulh x7,x13,x17 + umulh x8,x14,x17 + umulh x9,x15,x17 + umulh x10,x16,x17 + ldr x17,[x2,8*1] + + str x19,[x0] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,xzr, x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(1+1)] + adc x25,xzr,xzr + + str x19,[x0,8*1] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(2+1)] + adc x25,xzr,xzr + + str x19,[x0,8*2] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(3+1)] + adc x25,xzr,xzr + + str x19,[x0,8*3] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(4+1)] + adc x25,xzr,xzr + + str x19,[x0,8*4] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + adc x25,xzr,xzr + + str x19,[x0,8*5] + adds x19,x20,x5 + adcs x20,x21,x6 + adcs x21,x22,x7 + adcs x22,x23,x8 + adcs x23,x24,x9 + adc x24,x25,x10 + + stp x19,x20,[x0,#48] + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ret + + +.globl mul_382x + +.def mul_382x; +.type 32; +.endef +.p2align 5 +mul_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for two 384-bit vectors + + ldp x11,x12,[x1] + mov x26,x0 // save r_ptr + ldp x19,x20,[x1,#48] + mov x27,x1 // save a_ptr + ldp x13,x14,[x1,#16] + mov x28,x2 // save b_ptr + ldp x21,x22,[x1,#64] + ldp x15,x16,[x1,#32] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x23,x24,[x1,#80] + adcs x6,x12,x20 + ldp x11,x12,[x2] + adcs x7,x13,x21 + ldp x19,x20,[x2,#48] + adcs x8,x14,x22 + ldp x13,x14,[x2,#16] + adcs x9,x15,x23 + ldp x21,x22,[x2,#64] + adc x10,x16,x24 + ldp x15,x16,[x2,#32] + + stp x5,x6,[sp] + adds x5,x11,x19 // t1 = b->re + b->im + ldp x23,x24,[x2,#80] + adcs x6,x12,x20 + stp x7,x8,[sp,#16] + adcs x7,x13,x21 + adcs x8,x14,x22 + stp x9,x10,[sp,#32] + adcs x9,x15,x23 + stp x5,x6,[sp,#48] + adc x10,x16,x24 + stp x7,x8,[sp,#64] + stp x9,x10,[sp,#80] + + bl __mul_384 // mul_384(ret->re, a->re, b->re) + + add x1,sp,#0 // mul_384(ret->im, t0, t1) + add x2,sp,#48 + add x0,x26,#96 + bl __mul_384 + + add x1,x27,#48 // mul_384(tx, a->im, b->im) + add x2,x28,#48 + add x0,sp,#0 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + add x1,x26,#96 // ret->im -= tx + add x2,sp,#0 + add x0,x26,#96 + bl __sub_mod_384x384 + + add x2,x26,#0 // ret->im -= ret->re + bl __sub_mod_384x384 + + add x1,x26,#0 // ret->re -= tx + add x2,sp,#0 + add x0,x26,#0 + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl sqr_382x + +.def sqr_382x; +.type 32; +.endef +.p2align 5 +sqr_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x19,x20,[x1,#48] + ldp x13,x14,[x1,#16] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x21,x22,[x1,#64] + adcs x6,x12,x20 + ldp x15,x16,[x1,#32] + adcs x7,x13,x21 + ldp x23,x24,[x1,#80] + adcs x8,x14,x22 + stp x5,x6,[x0] + adcs x9,x15,x23 + ldp x5,x6,[x2] + adc x10,x16,x24 + stp x7,x8,[x0,#16] + + subs x11,x11,x19 // t1 = a->re - a->im + ldp x7,x8,[x2,#16] + sbcs x12,x12,x20 + stp x9,x10,[x0,#32] + sbcs x13,x13,x21 + ldp x9,x10,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + adds x11,x11,x19 + and x21,x7,x25 + adcs x12,x12,x20 + and x22,x8,x25 + adcs x13,x13,x21 + and x23,x9,x25 + adcs x14,x14,x22 + and x24,x10,x25 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + mov x4,x1 // save a_ptr + add x1,x0,#0 // mul_384(ret->re, t0, t1) + add x2,x0,#48 + bl __mul_384 + + add x1,x4,#0 // mul_384(ret->im, a->re, a->im) + add x2,x4,#48 + add x0,x0,#96 + bl __mul_384 + ldr x30,[x29,#8] + + ldp x11,x12,[x0] + ldp x13,x14,[x0,#16] + adds x11,x11,x11 // add with itself + ldp x15,x16,[x0,#32] + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adcs x19,x19,x19 + adcs x20,x20,x20 + stp x11,x12,[x0] + adcs x21,x21,x21 + stp x13,x14,[x0,#16] + adcs x22,x22,x22 + stp x15,x16,[x0,#32] + adcs x23,x23,x23 + stp x19,x20,[x0,#48] + adc x24,x24,x24 + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl sqr_mont_382x + +.def sqr_mont_382x; +.type 32; +.endef +.p2align 5 +sqr_mont_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#112 // space for two 384-bit vectors + word + mov x4,x3 // adjust for missing b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x17,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x5,x11,x17 // t0 = a->re + a->im + adcs x6,x12,x20 + adcs x7,x13,x21 + adcs x8,x14,x22 + adcs x9,x15,x23 + adc x10,x16,x24 + + subs x19,x11,x17 // t1 = a->re - a->im + sbcs x20,x12,x20 + sbcs x21,x13,x21 + sbcs x22,x14,x22 + sbcs x23,x15,x23 + sbcs x24,x16,x24 + sbc x25,xzr,xzr // borrow flag as mask + + stp x5,x6,[sp] + stp x7,x8,[sp,#16] + stp x9,x10,[sp,#32] + stp x19,x20,[sp,#48] + stp x21,x22,[sp,#64] + stp x23,x24,[sp,#80] + str x25,[sp,#96] + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) + + adds x19,x11,x11 // add with itself + adcs x20,x12,x12 + adcs x21,x13,x13 + adcs x22,x14,x14 + adcs x23,x15,x15 + adc x24,x16,x16 + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + ldp x11,x12,[sp] + ldr x17,[sp,#48] + ldp x13,x14,[sp,#16] + ldp x15,x16,[sp,#32] + + add x2,sp,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) + ldr x30,[x29,#8] + + ldr x25,[sp,#96] // account for sign from a->re - a->im + ldp x19,x20,[sp] + ldp x21,x22,[sp,#16] + ldp x23,x24,[sp,#32] + + and x19,x19,x25 + and x20,x20,x25 + and x21,x21,x25 + and x22,x22,x25 + and x23,x23,x25 + and x24,x24,x25 + + subs x11,x11,x19 + sbcs x12,x12,x20 + sbcs x13,x13,x21 + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + and x21,x7,x25 + and x22,x8,x25 + and x23,x9,x25 + and x24,x10,x25 + + adds x11,x11,x19 + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#112 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.def __mul_mont_383_nonred; +.type 32; +.endef +.p2align 5 +__mul_mont_383_nonred: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + ldr x17,[x2,8*1] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*2] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*3] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*4] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*5] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + + adds x11,x20,x26 + adcs x12,x21,x27 + adcs x13,x22,x28 + adcs x14,x23,x0 + adcs x15,x24,x1 + adcs x16,x25,x3 + + ret + + +.globl sgn0_pty_mont_384 + +.def sgn0_pty_mont_384; +.type 32; +.endef +.p2align 5 +sgn0_pty_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + adds x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl sgn0_pty_mont_384x + +.def sgn0_pty_mont_384x; +.type 32; +.endef +.p2align 5 +sgn0_pty_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + add x1,x1,#48 + + and x2,x11,#1 + orr x3,x11,x12 + adds x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + orr x3,x3,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x2,x2,x17 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + orr x1,x11,x12 + adds x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + orr x1,x1,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + diff --git a/crypto/blst_src/build/coff/mulq_mont_256-x86_64.s b/crypto/blst_src/build/coff/mulq_mont_256-x86_64.s new file mode 100644 index 00000000000..dd1e00fa301 --- /dev/null +++ b/crypto/blst_src/build/coff/mulq_mont_256-x86_64.s @@ -0,0 +1,872 @@ +.text + +.globl mul_mont_sparse_256 + +.def mul_mont_sparse_256; .scl 2; .type 32; .endef +.p2align 5 +mul_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_mont_sparse_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_mul_mont_sparse_256: + + + movq 0(%rdx),%rax + movq 0(%rsi),%r13 + movq 8(%rsi),%r14 + movq 16(%rsi),%r12 + movq 24(%rsi),%rbp + movq %rdx,%rbx + + movq %rax,%r15 + mulq %r13 + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mul_mont_sparse_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_mont_sparse_256: + +.globl sqr_mont_sparse_256 + +.def sqr_mont_sparse_256; .scl 2; .type 32; .endef +.p2align 5 +sqr_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_mont_sparse_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_sqr_mont_sparse_256: + + + movq 0(%rsi),%rax + movq %rcx,%r8 + movq 8(%rsi),%r14 + movq %rdx,%rcx + movq 16(%rsi),%r12 + leaq (%rsi),%rbx + movq 24(%rsi),%rbp + + movq %rax,%r15 + mulq %rax + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqr_mont_sparse_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_mont_sparse_256: +.def __mulq_mont_sparse_256; .scl 3; .type 32; .endef +.p2align 5 +__mulq_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + + mulq %r14 + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r12 + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq 8(%rbx),%rax + adcq $0,%rdx + xorq %r14,%r14 + movq %rdx,%r13 + + movq %r9,%rdi + imulq %r8,%r9 + + + movq %rax,%r15 + mulq 0(%rsi) + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + xorq %r15,%r15 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r9,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rdi,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + addq %rdx,%r13 + adcq $0,%r14 + adcq $0,%r15 + movq %r10,%rdi + imulq %r8,%r10 + + + movq %rax,%r9 + mulq 0(%rsi) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + xorq %r9,%r9 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r10,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rdi,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r13 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + addq %rdx,%r14 + adcq $0,%r15 + adcq $0,%r9 + movq %r11,%rdi + imulq %r8,%r11 + + + movq %rax,%r10 + mulq 0(%rsi) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r9 + xorq %r10,%r10 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r11,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rdi,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + addq %rdx,%r15 + adcq $0,%r9 + adcq $0,%r10 + imulq %r8,%rax + movq 8(%rsp),%rsi + + + movq %rax,%r11 + mulq 0(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r12,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + movq %r14,%rbx + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rdx,%r9 + adcq $0,%r10 + + + + + movq %r15,%r12 + subq 0(%rcx),%r13 + sbbq 8(%rcx),%r14 + sbbq 16(%rcx),%r15 + movq %r9,%rbp + sbbq 24(%rcx),%r9 + sbbq $0,%r10 + + cmovcq %rax,%r13 + cmovcq %rbx,%r14 + cmovcq %r12,%r15 + movq %r13,0(%rsi) + cmovcq %rbp,%r9 + movq %r14,8(%rsi) + movq %r15,16(%rsi) + movq %r9,24(%rsi) + + .byte 0xf3,0xc3 + + +.globl from_mont_256 + +.def from_mont_256; .scl 2; .type 32; .endef +.p2align 5 +from_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_from_mont_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_from_mont_256: + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + + + + + movq %r14,%r10 + movq %r15,%r11 + movq %r9,%r12 + + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + sbbq 24(%rbx),%r9 + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_from_mont_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_from_mont_256: + +.globl redc_mont_256 + +.def redc_mont_256; .scl 2; .type 32; .endef +.p2align 5 +redc_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_redc_mont_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_redc_mont_256: + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + addq 32(%rsi),%r13 + adcq 40(%rsi),%r14 + movq %r13,%rax + adcq 48(%rsi),%r15 + movq %r14,%r10 + adcq 56(%rsi),%r9 + sbbq %rsi,%rsi + + + + + movq %r15,%r11 + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + movq %r9,%r12 + sbbq 24(%rbx),%r9 + sbbq $0,%rsi + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_redc_mont_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_redc_mont_256: +.def __mulq_by_1_mont_256; .scl 3; .type 32; .endef +.p2align 5 +__mulq_by_1_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + + movq %rax,%r13 + imulq %rcx,%rax + movq %rax,%r9 + + mulq 0(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq %rdx,%r13 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r10 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 16(%rbx) + movq %r10,%r14 + imulq %rcx,%r10 + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r11 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r13,%r12 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r9 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_mul_mont_sparse_256 +.rva .LSEH_body_mul_mont_sparse_256 +.rva .LSEH_info_mul_mont_sparse_256_prologue + +.rva .LSEH_body_mul_mont_sparse_256 +.rva .LSEH_epilogue_mul_mont_sparse_256 +.rva .LSEH_info_mul_mont_sparse_256_body + +.rva .LSEH_epilogue_mul_mont_sparse_256 +.rva .LSEH_end_mul_mont_sparse_256 +.rva .LSEH_info_mul_mont_sparse_256_epilogue + +.rva .LSEH_begin_sqr_mont_sparse_256 +.rva .LSEH_body_sqr_mont_sparse_256 +.rva .LSEH_info_sqr_mont_sparse_256_prologue + +.rva .LSEH_body_sqr_mont_sparse_256 +.rva .LSEH_epilogue_sqr_mont_sparse_256 +.rva .LSEH_info_sqr_mont_sparse_256_body + +.rva .LSEH_epilogue_sqr_mont_sparse_256 +.rva .LSEH_end_sqr_mont_sparse_256 +.rva .LSEH_info_sqr_mont_sparse_256_epilogue + +.rva .LSEH_begin_from_mont_256 +.rva .LSEH_body_from_mont_256 +.rva .LSEH_info_from_mont_256_prologue + +.rva .LSEH_body_from_mont_256 +.rva .LSEH_epilogue_from_mont_256 +.rva .LSEH_info_from_mont_256_body + +.rva .LSEH_epilogue_from_mont_256 +.rva .LSEH_end_from_mont_256 +.rva .LSEH_info_from_mont_256_epilogue + +.rva .LSEH_begin_redc_mont_256 +.rva .LSEH_body_redc_mont_256 +.rva .LSEH_info_redc_mont_256_prologue + +.rva .LSEH_body_redc_mont_256 +.rva .LSEH_epilogue_redc_mont_256 +.rva .LSEH_info_redc_mont_256_body + +.rva .LSEH_epilogue_redc_mont_256 +.rva .LSEH_end_redc_mont_256 +.rva .LSEH_info_redc_mont_256_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_mul_mont_sparse_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_mont_sparse_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mul_mont_sparse_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_mont_sparse_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_mont_sparse_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqr_mont_sparse_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_from_mont_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_from_mont_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_from_mont_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_redc_mont_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_redc_mont_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_redc_mont_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/mulq_mont_384-x86_64.s b/crypto/blst_src/build/coff/mulq_mont_384-x86_64.s new file mode 100644 index 00000000000..5663463524a --- /dev/null +++ b/crypto/blst_src/build/coff/mulq_mont_384-x86_64.s @@ -0,0 +1,4206 @@ +.text + + + + + + + +.def __sub_mod_384x384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 + + +.def __add_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__add_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + + +.def __sub_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + +.globl mul_mont_384x + +.def mul_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +mul_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $328,%rsp + +.LSEH_body_mul_mont_384x: + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulq_384 + + + leaq 48(%rbx),%rbx + leaq 48(%rsi),%rsi + leaq 40+96(%rsp),%rdi + call __mulq_384 + + + movq 8(%rsp),%rcx + leaq -48(%rsi),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulq_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + movq %rcx,%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_mul_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_mont_384x: +.globl sqr_mont_384x + +.def sqr_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +sqr_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqr_mont_384x: + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + call __mulq_mont_384 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + movq %r14,%r12 + adcq %r9,%r9 + movq %r15,%r13 + adcq %r10,%r10 + movq %r8,%rax + adcq %r11,%r11 + movq %r9,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r10,%rbp + sbbq 16(%rcx),%r8 + sbbq 24(%rcx),%r9 + sbbq 32(%rcx),%r10 + movq %r11,%rsi + sbbq 40(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %r12,%r14 + cmovcq %r13,%r15 + cmovcq %rax,%r8 + movq %r14,48(%rdi) + cmovcq %rbx,%r9 + movq %r15,56(%rdi) + cmovcq %rbp,%r10 + movq %r8,64(%rdi) + cmovcq %rsi,%r11 + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqr_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_mont_384x: + +.globl mul_382x + +.def mul_382x; .scl 2; .type 32; .endef +.p2align 5 +mul_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_mul_382x: + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulq_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulq_384 + + + leaq 48(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulq_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_mul_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_382x: +.globl sqr_382x + +.def sqr_382x; .scl 2; .type 32; .endef +.p2align 5 +sqr_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rsi + +.LSEH_body_sqr_382x: + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulq_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulq_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqr_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_382x: +.globl mul_384 + +.def mul_384; .scl 2; .type 32; .endef +.p2align 5 +mul_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + +.LSEH_body_mul_384: + + + movq %rdx,%rbx + call __mulq_384 + + movq 0(%rsp),%r12 + + movq 8(%rsp),%rbx + + movq 16(%rsp),%rbp + + leaq 24(%rsp),%rsp + +.LSEH_epilogue_mul_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_384: + +.def __mulq_384; .scl 3; .type 32; .endef +.p2align 5 +__mulq_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rax + + movq %rax,%rbp + mulq 0(%rsi) + movq %rax,0(%rdi) + movq %rbp,%rax + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r11 + movq 8(%rbx),%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,8(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,16(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,24(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,32(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,40(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq %rax,%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rcx,48(%rdi) + movq %r8,56(%rdi) + movq %r9,64(%rdi) + movq %r10,72(%rdi) + movq %r11,80(%rdi) + movq %r12,88(%rdi) + + .byte 0xf3,0xc3 + +.globl sqr_384 + +.def sqr_384; .scl 2; .type 32; .endef +.p2align 5 +sqr_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_384: + movq %rcx,%rdi + movq %rdx,%rsi + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sqr_384: + + + call __sqrq_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqr_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_384: + +.def __sqrq_384; .scl 3; .type 32; .endef +.p2align 5 +__sqrq_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r15 + movq 16(%rsi),%rcx + movq 24(%rsi),%rbx + + + movq %rax,%r14 + mulq %r15 + movq %rax,%r9 + movq %r14,%rax + movq 32(%rsi),%rbp + movq %rdx,%r10 + + mulq %rcx + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + movq 40(%rsi),%rsi + movq %rdx,%r11 + + mulq %rbx + addq %rax,%r11 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq %rax + xorq %r8,%r8 + movq %rax,0(%rdi) + movq %r15,%rax + addq %r9,%r9 + adcq $0,%r8 + addq %rdx,%r9 + adcq $0,%r8 + movq %r9,8(%rdi) + + mulq %rcx + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbx + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbp + addq %rax,%r13 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq %rax + xorq %r9,%r9 + addq %rax,%r8 + movq %rcx,%rax + addq %r10,%r10 + adcq %r11,%r11 + adcq $0,%r9 + addq %r8,%r10 + adcq %rdx,%r11 + adcq $0,%r9 + movq %r10,16(%rdi) + + mulq %rbx + addq %rax,%r13 + movq %rcx,%rax + adcq $0,%rdx + movq %r11,24(%rdi) + movq %rdx,%r8 + + mulq %rbp + addq %rax,%r14 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq %rsi + addq %rax,%r15 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + xorq %r11,%r11 + addq %rax,%r9 + movq %rbx,%rax + addq %r12,%r12 + adcq %r13,%r13 + adcq $0,%r11 + addq %r9,%r12 + adcq %rdx,%r13 + adcq $0,%r11 + movq %r12,32(%rdi) + + + mulq %rbp + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %r13,40(%rdi) + movq %rdx,%r8 + + mulq %rsi + addq %rax,%rcx + movq %rbx,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%rbx + + mulq %rax + xorq %r12,%r12 + addq %rax,%r11 + movq %rbp,%rax + addq %r14,%r14 + adcq %r15,%r15 + adcq $0,%r12 + addq %r11,%r14 + adcq %rdx,%r15 + movq %r14,48(%rdi) + adcq $0,%r12 + movq %r15,56(%rdi) + + + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq %rax + xorq %r13,%r13 + addq %rax,%r12 + movq %rsi,%rax + addq %rcx,%rcx + adcq %rbx,%rbx + adcq $0,%r13 + addq %r12,%rcx + adcq %rdx,%rbx + movq %rcx,64(%rdi) + adcq $0,%r13 + movq %rbx,72(%rdi) + + + mulq %rax + addq %r13,%rax + addq %rbp,%rbp + adcq $0,%rdx + addq %rbp,%rax + adcq $0,%rdx + movq %rax,80(%rdi) + movq %rdx,88(%rdi) + + .byte 0xf3,0xc3 + + +.globl sqr_mont_384 + +.def sqr_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sqr_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $120,%rsp + +.LSEH_body_sqr_mont_384: + + + movq %rcx,96(%rsp) + movq %rdx,104(%rsp) + movq %rdi,112(%rsp) + + movq %rsp,%rdi + call __sqrq_384 + + leaq 0(%rsp),%rsi + movq 96(%rsp),%rcx + movq 104(%rsp),%rbx + movq 112(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 120(%rsp),%r8 + movq 120(%rsp),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqr_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_mont_384: + + + +.globl redc_mont_384 + +.def redc_mont_384; .scl 2; .type 32; .endef +.p2align 5 +redc_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_redc_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_redc_mont_384: + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_redc_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_redc_mont_384: + + + + +.globl from_mont_384 + +.def from_mont_384; .scl 2; .type 32; .endef +.p2align 5 +from_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_from_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_from_mont_384: + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + + + + + + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_from_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_from_mont_384: +.def __mulq_by_1_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__mulq_by_1_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r8 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r8,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r9 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r10 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %r9,%r15 + imulq %rcx,%r9 + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 32(%rbx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 40(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r9,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %r10,%r8 + imulq %rcx,%r10 + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r8 + movq %r10,%rax + adcq %rdx,%r8 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %r11,%r9 + imulq %rcx,%r11 + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r11,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %r12,%r10 + imulq %rcx,%r12 + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %r13,%r11 + imulq %rcx,%r13 + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 + + +.def __redc_tail_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__redc_tail_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 + + +.globl sgn0_pty_mont_384 + +.def sgn0_pty_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sgn0_pty_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0_pty_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sgn0_pty_mont_384: + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sgn0_pty_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0_pty_mont_384: + +.globl sgn0_pty_mont_384x + +.def sgn0_pty_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +sgn0_pty_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0_pty_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sgn0_pty_mont_384x: + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sgn0_pty_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0_pty_mont_384x: +.globl mul_mont_384 + +.def mul_mont_384; .scl 2; .type 32; .endef +.p2align 5 +mul_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mul_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $24,%rsp + +.LSEH_body_mul_mont_384: + + + movq 0(%rdx),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + movq %rdx,%rbx + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + + call __mulq_mont_384 + + movq 24(%rsp),%r15 + + movq 32(%rsp),%r14 + + movq 40(%rsp),%r13 + + movq 48(%rsp),%r12 + + movq 56(%rsp),%rbx + + movq 64(%rsp),%rbp + + leaq 72(%rsp),%rsp + +.LSEH_epilogue_mul_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mul_mont_384: +.def __mulq_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__mulq_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rdi + mulq %r14 + movq %rax,%r8 + movq %rdi,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%rbp + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + xorq %r15,%r15 + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r8,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + adcq $0,%r15 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r9,%rbp + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r14 + adcq $0,%rdx + xorq %r8,%r8 + addq %rax,%r14 + movq %r9,%rax + adcq %rdx,%r15 + adcq $0,%r8 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r9,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + adcq $0,%r8 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r10,%rbp + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r15 + adcq $0,%rdx + xorq %r9,%r9 + addq %rax,%r15 + movq %r10,%rax + adcq %rdx,%r8 + adcq $0,%r9 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r10,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r8 + adcq $0,%r9 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r11,%rbp + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r8 + adcq $0,%rdx + xorq %r10,%r10 + addq %rax,%r8 + movq %r11,%rax + adcq %rdx,%r9 + adcq $0,%r10 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r11,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq %rdx,%r9 + adcq $0,%r10 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r12,%rbp + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r9 + adcq $0,%rdx + xorq %r11,%r11 + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r10 + adcq $0,%r11 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r12,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq %rdx,%r10 + adcq $0,%r11 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + movq %r13,%rbp + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r8 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rsi) + addq %r12,%r10 + adcq $0,%rdx + xorq %r12,%r12 + addq %rax,%r10 + movq %r13,%rax + adcq %rdx,%r11 + adcq $0,%r12 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r13,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq %rdx,%r11 + adcq $0,%r12 + + + + + movq 16(%rsp),%rdi + subq 0(%rcx),%r14 + movq %r15,%rdx + sbbq 8(%rcx),%r15 + movq %r8,%rbx + sbbq 16(%rcx),%r8 + movq %r9,%rsi + sbbq 24(%rcx),%r9 + movq %r10,%rbp + sbbq 32(%rcx),%r10 + movq %r11,%r13 + sbbq 40(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rdx,%r15 + cmovcq %rbx,%r8 + movq %r14,0(%rdi) + cmovcq %rsi,%r9 + movq %r15,8(%rdi) + cmovcq %rbp,%r10 + movq %r8,16(%rdi) + cmovcq %r13,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 + +.globl sqr_n_mul_mont_384 + +.def sqr_n_mul_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sqr_n_mul_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_n_mul_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + movq 48(%rsp),%r9 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqr_n_mul_mont_384: + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +.Loop_sqr_384: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movd %xmm1,%edx + leaq 0(%rdi),%rsi + decl %edx + jnz .Loop_sqr_384 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqr_n_mul_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_n_mul_mont_384: + +.globl sqr_n_mul_mont_383 + +.def sqr_n_mul_mont_383; .scl 2; .type 32; .endef +.p2align 5 +sqr_n_mul_mont_383: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_n_mul_mont_383: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + movq 48(%rsp),%r9 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqr_n_mul_mont_383: + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +.Loop_sqr_383: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + + movd %xmm1,%edx + addq 48(%rsi),%r14 + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + leaq 0(%rdi),%rsi + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + decl %edx + jnz .Loop_sqr_383 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqr_n_mul_mont_383: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_n_mul_mont_383: +.def __mulq_mont_383_nonred; .scl 3; .type 32; .endef +.p2align 5 +__mulq_mont_383_nonred: + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rbp + mulq %r14 + movq %rax,%r8 + movq %rbp,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%r15 + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%r15 + movq %r8,%rax + adcq %rdx,%r15 + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r9 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rcx) + addq %r15,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %r15,%r13 + adcq %rdx,%r14 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + movq %r9,%r8 + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rsi) + addq %r15,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rcx) + addq %rax,%r8 + movq %r9,%rax + adcq %rdx,%r8 + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rcx) + addq %r8,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r8,%r14 + adcq %rdx,%r15 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r10,%r9 + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rcx) + addq %rax,%r9 + movq %r10,%rax + adcq %rdx,%r9 + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rcx) + addq %r9,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r9,%r15 + adcq %rdx,%r8 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r11,%r10 + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rcx) + addq %rax,%r10 + movq %r11,%rax + adcq %rdx,%r10 + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rcx) + addq %r10,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r10,%r8 + adcq %rdx,%r9 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r12,%r11 + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r9 + adcq $0,%rdx + addq %rax,%r9 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rcx) + addq %rax,%r11 + movq %r12,%rax + adcq %rdx,%r11 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rcx) + addq %r11,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r11,%r9 + adcq %rdx,%r10 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r13,%r12 + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r10 + adcq $0,%rdx + addq %rax,%r10 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 0(%rcx) + addq %rax,%r12 + movq %r13,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 24(%rcx) + addq %r12,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r12,%r10 + adcq %rdx,%r11 + .byte 0xf3,0xc3 + +.globl sqr_mont_382x + +.def sqr_mont_382x; .scl 2; .type 32; .endef +.p2align 5 +sqr_mont_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqr_mont_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqr_mont_382x: + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rsi,16(%rsp) + movq %rdi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq 24(%rsp),%rdi + call __mulq_mont_383_nonred + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %r8,64(%rdi) + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_383_nonred + movq 32+96(%rsp),%rsi + movq 32+0(%rsp),%r12 + movq 32+8(%rsp),%r13 + andq %rsi,%r12 + movq 32+16(%rsp),%rax + andq %rsi,%r13 + movq 32+24(%rsp),%rbx + andq %rsi,%rax + movq 32+32(%rsp),%rbp + andq %rsi,%rbx + andq %rsi,%rbp + andq 32+40(%rsp),%rsi + + subq %r12,%r14 + movq 0(%rcx),%r12 + sbbq %r13,%r15 + movq 8(%rcx),%r13 + sbbq %rax,%r8 + movq 16(%rcx),%rax + sbbq %rbx,%r9 + movq 24(%rcx),%rbx + sbbq %rbp,%r10 + movq 32(%rcx),%rbp + sbbq %rsi,%r11 + sbbq %rsi,%rsi + + andq %rsi,%r12 + andq %rsi,%r13 + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r12,%r14 + adcq %r13,%r15 + adcq %rax,%r8 + adcq %rbx,%r9 + adcq %rbp,%r10 + adcq %rsi,%r11 + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqr_mont_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqr_mont_382x: +.section .pdata +.p2align 2 +.rva .LSEH_begin_mul_mont_384x +.rva .LSEH_body_mul_mont_384x +.rva .LSEH_info_mul_mont_384x_prologue + +.rva .LSEH_body_mul_mont_384x +.rva .LSEH_epilogue_mul_mont_384x +.rva .LSEH_info_mul_mont_384x_body + +.rva .LSEH_epilogue_mul_mont_384x +.rva .LSEH_end_mul_mont_384x +.rva .LSEH_info_mul_mont_384x_epilogue + +.rva .LSEH_begin_sqr_mont_384x +.rva .LSEH_body_sqr_mont_384x +.rva .LSEH_info_sqr_mont_384x_prologue + +.rva .LSEH_body_sqr_mont_384x +.rva .LSEH_epilogue_sqr_mont_384x +.rva .LSEH_info_sqr_mont_384x_body + +.rva .LSEH_epilogue_sqr_mont_384x +.rva .LSEH_end_sqr_mont_384x +.rva .LSEH_info_sqr_mont_384x_epilogue + +.rva .LSEH_begin_mul_382x +.rva .LSEH_body_mul_382x +.rva .LSEH_info_mul_382x_prologue + +.rva .LSEH_body_mul_382x +.rva .LSEH_epilogue_mul_382x +.rva .LSEH_info_mul_382x_body + +.rva .LSEH_epilogue_mul_382x +.rva .LSEH_end_mul_382x +.rva .LSEH_info_mul_382x_epilogue + +.rva .LSEH_begin_sqr_382x +.rva .LSEH_body_sqr_382x +.rva .LSEH_info_sqr_382x_prologue + +.rva .LSEH_body_sqr_382x +.rva .LSEH_epilogue_sqr_382x +.rva .LSEH_info_sqr_382x_body + +.rva .LSEH_epilogue_sqr_382x +.rva .LSEH_end_sqr_382x +.rva .LSEH_info_sqr_382x_epilogue + +.rva .LSEH_begin_mul_384 +.rva .LSEH_body_mul_384 +.rva .LSEH_info_mul_384_prologue + +.rva .LSEH_body_mul_384 +.rva .LSEH_epilogue_mul_384 +.rva .LSEH_info_mul_384_body + +.rva .LSEH_epilogue_mul_384 +.rva .LSEH_end_mul_384 +.rva .LSEH_info_mul_384_epilogue + +.rva .LSEH_begin_sqr_384 +.rva .LSEH_body_sqr_384 +.rva .LSEH_info_sqr_384_prologue + +.rva .LSEH_body_sqr_384 +.rva .LSEH_epilogue_sqr_384 +.rva .LSEH_info_sqr_384_body + +.rva .LSEH_epilogue_sqr_384 +.rva .LSEH_end_sqr_384 +.rva .LSEH_info_sqr_384_epilogue + +.rva .LSEH_begin_sqr_mont_384 +.rva .LSEH_body_sqr_mont_384 +.rva .LSEH_info_sqr_mont_384_prologue + +.rva .LSEH_body_sqr_mont_384 +.rva .LSEH_epilogue_sqr_mont_384 +.rva .LSEH_info_sqr_mont_384_body + +.rva .LSEH_epilogue_sqr_mont_384 +.rva .LSEH_end_sqr_mont_384 +.rva .LSEH_info_sqr_mont_384_epilogue + +.rva .LSEH_begin_redc_mont_384 +.rva .LSEH_body_redc_mont_384 +.rva .LSEH_info_redc_mont_384_prologue + +.rva .LSEH_body_redc_mont_384 +.rva .LSEH_epilogue_redc_mont_384 +.rva .LSEH_info_redc_mont_384_body + +.rva .LSEH_epilogue_redc_mont_384 +.rva .LSEH_end_redc_mont_384 +.rva .LSEH_info_redc_mont_384_epilogue + +.rva .LSEH_begin_from_mont_384 +.rva .LSEH_body_from_mont_384 +.rva .LSEH_info_from_mont_384_prologue + +.rva .LSEH_body_from_mont_384 +.rva .LSEH_epilogue_from_mont_384 +.rva .LSEH_info_from_mont_384_body + +.rva .LSEH_epilogue_from_mont_384 +.rva .LSEH_end_from_mont_384 +.rva .LSEH_info_from_mont_384_epilogue + +.rva .LSEH_begin_sgn0_pty_mont_384 +.rva .LSEH_body_sgn0_pty_mont_384 +.rva .LSEH_info_sgn0_pty_mont_384_prologue + +.rva .LSEH_body_sgn0_pty_mont_384 +.rva .LSEH_epilogue_sgn0_pty_mont_384 +.rva .LSEH_info_sgn0_pty_mont_384_body + +.rva .LSEH_epilogue_sgn0_pty_mont_384 +.rva .LSEH_end_sgn0_pty_mont_384 +.rva .LSEH_info_sgn0_pty_mont_384_epilogue + +.rva .LSEH_begin_sgn0_pty_mont_384x +.rva .LSEH_body_sgn0_pty_mont_384x +.rva .LSEH_info_sgn0_pty_mont_384x_prologue + +.rva .LSEH_body_sgn0_pty_mont_384x +.rva .LSEH_epilogue_sgn0_pty_mont_384x +.rva .LSEH_info_sgn0_pty_mont_384x_body + +.rva .LSEH_epilogue_sgn0_pty_mont_384x +.rva .LSEH_end_sgn0_pty_mont_384x +.rva .LSEH_info_sgn0_pty_mont_384x_epilogue + +.rva .LSEH_begin_mul_mont_384 +.rva .LSEH_body_mul_mont_384 +.rva .LSEH_info_mul_mont_384_prologue + +.rva .LSEH_body_mul_mont_384 +.rva .LSEH_epilogue_mul_mont_384 +.rva .LSEH_info_mul_mont_384_body + +.rva .LSEH_epilogue_mul_mont_384 +.rva .LSEH_end_mul_mont_384 +.rva .LSEH_info_mul_mont_384_epilogue + +.rva .LSEH_begin_sqr_n_mul_mont_384 +.rva .LSEH_body_sqr_n_mul_mont_384 +.rva .LSEH_info_sqr_n_mul_mont_384_prologue + +.rva .LSEH_body_sqr_n_mul_mont_384 +.rva .LSEH_epilogue_sqr_n_mul_mont_384 +.rva .LSEH_info_sqr_n_mul_mont_384_body + +.rva .LSEH_epilogue_sqr_n_mul_mont_384 +.rva .LSEH_end_sqr_n_mul_mont_384 +.rva .LSEH_info_sqr_n_mul_mont_384_epilogue + +.rva .LSEH_begin_sqr_n_mul_mont_383 +.rva .LSEH_body_sqr_n_mul_mont_383 +.rva .LSEH_info_sqr_n_mul_mont_383_prologue + +.rva .LSEH_body_sqr_n_mul_mont_383 +.rva .LSEH_epilogue_sqr_n_mul_mont_383 +.rva .LSEH_info_sqr_n_mul_mont_383_body + +.rva .LSEH_epilogue_sqr_n_mul_mont_383 +.rva .LSEH_end_sqr_n_mul_mont_383 +.rva .LSEH_info_sqr_n_mul_mont_383_epilogue + +.rva .LSEH_begin_sqr_mont_382x +.rva .LSEH_body_sqr_mont_382x +.rva .LSEH_info_sqr_mont_382x_prologue + +.rva .LSEH_body_sqr_mont_382x +.rva .LSEH_epilogue_sqr_mont_382x +.rva .LSEH_info_sqr_mont_382x_body + +.rva .LSEH_epilogue_sqr_mont_382x +.rva .LSEH_end_sqr_mont_382x +.rva .LSEH_info_sqr_mont_382x_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_mul_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_mont_384x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x29,0x00 +.byte 0x00,0xe4,0x2a,0x00 +.byte 0x00,0xd4,0x2b,0x00 +.byte 0x00,0xc4,0x2c,0x00 +.byte 0x00,0x34,0x2d,0x00 +.byte 0x00,0x54,0x2e,0x00 +.byte 0x00,0x74,0x30,0x00 +.byte 0x00,0x64,0x31,0x00 +.byte 0x00,0x01,0x2f,0x00 +.LSEH_info_mul_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_mont_384x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqr_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_382x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_mul_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_382x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqr_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_384_body: +.byte 1,0,11,0 +.byte 0x00,0xc4,0x00,0x00 +.byte 0x00,0x34,0x01,0x00 +.byte 0x00,0x54,0x02,0x00 +.byte 0x00,0x74,0x04,0x00 +.byte 0x00,0x64,0x05,0x00 +.byte 0x00,0x22 +.byte 0x00,0x00,0x00,0x00,0x00,0x00 +.LSEH_info_mul_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqr_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_mont_384_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x0f,0x00 +.byte 0x00,0xe4,0x10,0x00 +.byte 0x00,0xd4,0x11,0x00 +.byte 0x00,0xc4,0x12,0x00 +.byte 0x00,0x34,0x13,0x00 +.byte 0x00,0x54,0x14,0x00 +.byte 0x00,0x74,0x16,0x00 +.byte 0x00,0x64,0x17,0x00 +.byte 0x00,0x01,0x15,0x00 +.LSEH_info_sqr_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_redc_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_redc_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_redc_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_from_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_from_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_from_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0_pty_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0_pty_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sgn0_pty_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0_pty_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0_pty_mont_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sgn0_pty_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mul_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mul_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x03,0x00 +.byte 0x00,0xe4,0x04,0x00 +.byte 0x00,0xd4,0x05,0x00 +.byte 0x00,0xc4,0x06,0x00 +.byte 0x00,0x34,0x07,0x00 +.byte 0x00,0x54,0x08,0x00 +.byte 0x00,0x74,0x0a,0x00 +.byte 0x00,0x64,0x0b,0x00 +.byte 0x00,0x82 +.byte 0x00,0x00 +.LSEH_info_mul_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_n_mul_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_n_mul_mont_384_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqr_n_mul_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_n_mul_mont_383_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_n_mul_mont_383_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqr_n_mul_mont_383_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqr_mont_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqr_mont_382x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqr_mont_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/mulx_mont_256-x86_64.s b/crypto/blst_src/build/coff/mulx_mont_256-x86_64.s new file mode 100644 index 00000000000..75c7e82bc1a --- /dev/null +++ b/crypto/blst_src/build/coff/mulx_mont_256-x86_64.s @@ -0,0 +1,784 @@ +.text + +.globl mulx_mont_sparse_256 + +.def mulx_mont_sparse_256; .scl 2; .type 32; .endef +.p2align 5 +mulx_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mulx_mont_sparse_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_mulx_mont_sparse_256: + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_mulx_mont_sparse_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mulx_mont_sparse_256: + +.globl sqrx_mont_sparse_256 + +.def sqrx_mont_sparse_256; .scl 2; .type 32; .endef +.p2align 5 +sqrx_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_mont_sparse_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sqrx_mont_sparse_256: + + + movq %rsi,%rbx + movq %rcx,%r8 + movq %rdx,%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqrx_mont_sparse_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_mont_sparse_256: +.def __mulx_mont_sparse_256; .scl 3; .type 32; .endef +.p2align 5 +__mulx_mont_sparse_256: + .byte 0xf3,0x0f,0x1e,0xfa + + mulxq %r15,%r15,%r12 + mulxq %rbp,%rbp,%r13 + addq %r15,%r11 + mulxq %r9,%r9,%r14 + movq 8(%rbx),%rdx + adcq %rbp,%r12 + adcq %r9,%r13 + adcq $0,%r14 + + movq %rax,%r10 + imulq %r8,%rax + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r11 + adcxq %r9,%r12 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r14 + adcxq %r15,%r9 + adoxq %r9,%r15 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r10 + adoxq %r11,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r12 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r12 + adoxq %r9,%r13 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 16(%rbx),%rdx + adcxq %rbp,%r13 + adoxq %r9,%r14 + adcxq %r10,%r14 + adoxq %r10,%r15 + adcxq %r10,%r15 + adoxq %r10,%r10 + adcq $0,%r10 + movq %rax,%r11 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r15 + adcxq %r10,%r9 + adoxq %r9,%r10 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r11 + adoxq %r12,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r13 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r13 + adoxq %r9,%r14 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 24(%rbx),%rdx + adcxq %rbp,%r14 + adoxq %r9,%r15 + adcxq %r11,%r15 + adoxq %r11,%r10 + adcxq %r11,%r10 + adoxq %r11,%r11 + adcq $0,%r11 + movq %rax,%r12 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r15 + adcxq %r9,%r10 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r10 + adcxq %r11,%r9 + adoxq %r9,%r11 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r12 + adoxq %r13,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r14 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %rax,%rdx + adcxq %rbp,%r15 + adoxq %r9,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + adoxq %r12,%r12 + adcq $0,%r12 + imulq %r8,%rdx + + + xorq %rbp,%rbp + mulxq 0+128(%rcx),%r13,%r9 + adcxq %rax,%r13 + adoxq %r9,%r14 + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r15 + adoxq %r9,%r10 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %r14,%rdx + leaq 128(%rcx),%rcx + adcxq %rbp,%r10 + adoxq %r9,%r11 + movq %r15,%rax + adcxq %r13,%r11 + adoxq %r13,%r12 + adcq $0,%r12 + + + + + movq %r10,%rbp + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + sbbq 16(%rcx),%r10 + movq %r11,%r9 + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rdx,%r14 + cmovcq %rax,%r15 + cmovcq %rbp,%r10 + movq %r14,0(%rdi) + cmovcq %r9,%r11 + movq %r15,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + .byte 0xf3,0xc3 + +.globl fromx_mont_256 + +.def fromx_mont_256; .scl 2; .type 32; .endef +.p2align 5 +fromx_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_fromx_mont_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_fromx_mont_256: + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + + + + + movq %r15,%rdx + movq %r10,%r12 + movq %r11,%r13 + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + sbbq 24(%rbx),%r11 + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_fromx_mont_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_fromx_mont_256: + +.globl redcx_mont_256 + +.def redcx_mont_256; .scl 2; .type 32; .endef +.p2align 5 +redcx_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_redcx_mont_256: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_redcx_mont_256: + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + addq 32(%rsi),%r14 + adcq 40(%rsi),%r15 + movq %r14,%rax + adcq 48(%rsi),%r10 + movq %r15,%rdx + adcq 56(%rsi),%r11 + sbbq %rsi,%rsi + + + + + movq %r10,%r12 + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + movq %r11,%r13 + sbbq 24(%rbx),%r11 + sbbq $0,%rsi + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_redcx_mont_256: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_redcx_mont_256: +.def __mulx_by_1_mont_256; .scl 3; .type 32; .endef +.p2align 5 +__mulx_by_1_mont_256: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r11 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r10 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r10 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + movq %r13,%r11 + imulq %rcx,%r13 + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_mulx_mont_sparse_256 +.rva .LSEH_body_mulx_mont_sparse_256 +.rva .LSEH_info_mulx_mont_sparse_256_prologue + +.rva .LSEH_body_mulx_mont_sparse_256 +.rva .LSEH_epilogue_mulx_mont_sparse_256 +.rva .LSEH_info_mulx_mont_sparse_256_body + +.rva .LSEH_epilogue_mulx_mont_sparse_256 +.rva .LSEH_end_mulx_mont_sparse_256 +.rva .LSEH_info_mulx_mont_sparse_256_epilogue + +.rva .LSEH_begin_sqrx_mont_sparse_256 +.rva .LSEH_body_sqrx_mont_sparse_256 +.rva .LSEH_info_sqrx_mont_sparse_256_prologue + +.rva .LSEH_body_sqrx_mont_sparse_256 +.rva .LSEH_epilogue_sqrx_mont_sparse_256 +.rva .LSEH_info_sqrx_mont_sparse_256_body + +.rva .LSEH_epilogue_sqrx_mont_sparse_256 +.rva .LSEH_end_sqrx_mont_sparse_256 +.rva .LSEH_info_sqrx_mont_sparse_256_epilogue + +.rva .LSEH_begin_fromx_mont_256 +.rva .LSEH_body_fromx_mont_256 +.rva .LSEH_info_fromx_mont_256_prologue + +.rva .LSEH_body_fromx_mont_256 +.rva .LSEH_epilogue_fromx_mont_256 +.rva .LSEH_info_fromx_mont_256_body + +.rva .LSEH_epilogue_fromx_mont_256 +.rva .LSEH_end_fromx_mont_256 +.rva .LSEH_info_fromx_mont_256_epilogue + +.rva .LSEH_begin_redcx_mont_256 +.rva .LSEH_body_redcx_mont_256 +.rva .LSEH_info_redcx_mont_256_prologue + +.rva .LSEH_body_redcx_mont_256 +.rva .LSEH_epilogue_redcx_mont_256 +.rva .LSEH_info_redcx_mont_256_body + +.rva .LSEH_epilogue_redcx_mont_256 +.rva .LSEH_end_redcx_mont_256 +.rva .LSEH_info_redcx_mont_256_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_mulx_mont_sparse_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mulx_mont_sparse_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_mulx_mont_sparse_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_mont_sparse_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_mont_sparse_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqrx_mont_sparse_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_fromx_mont_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_fromx_mont_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_fromx_mont_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_redcx_mont_256_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_redcx_mont_256_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_redcx_mont_256_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/mulx_mont_384-x86_64.s b/crypto/blst_src/build/coff/mulx_mont_384-x86_64.s new file mode 100644 index 00000000000..12306a7ff5c --- /dev/null +++ b/crypto/blst_src/build/coff/mulx_mont_384-x86_64.s @@ -0,0 +1,3559 @@ +.text + + + + + + + +.def __sub_mod_384x384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384x384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 + + +.def __add_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__add_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + + +.def __sub_mod_384; .scl 3; .type 32; .endef +.p2align 5 +__sub_mod_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 + +.globl mulx_mont_384x + +.def mulx_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +mulx_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mulx_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $328,%rsp + +.LSEH_body_mulx_mont_384x: + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulx_384 + + + leaq 48(%rbx),%rbx + leaq 128+48(%rsi),%rsi + leaq 96(%rdi),%rdi + call __mulx_384 + + + movq 8(%rsp),%rcx + leaq (%rbx),%rsi + leaq -48(%rbx),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulx_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + leaq (%rcx),%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_mulx_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mulx_mont_384x: +.globl sqrx_mont_384x + +.def sqrx_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +sqrx_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqrx_mont_384x: + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + movq %rdx,%r8 + adcq %r12,%r12 + movq %r15,%r9 + adcq %rdi,%rdi + movq %rax,%r10 + adcq %rbp,%rbp + movq %r12,%r11 + sbbq %rsi,%rsi + + subq 0(%rcx),%rdx + sbbq 8(%rcx),%r15 + movq %rdi,%r13 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r12 + sbbq 32(%rcx),%rdi + movq %rbp,%r14 + sbbq 40(%rcx),%rbp + sbbq $0,%rsi + + cmovcq %r8,%rdx + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %rdx,48(%rbx) + cmovcq %r11,%r12 + movq %r15,56(%rbx) + cmovcq %r13,%rdi + movq %rax,64(%rbx) + cmovcq %r14,%rbp + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqrx_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_mont_384x: + +.globl mulx_382x + +.def mulx_382x; .scl 2; .type 32; .endef +.p2align 5 +mulx_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mulx_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_mulx_382x: + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulx_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulx_384 + + + leaq 48+128(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulx_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_mulx_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mulx_382x: +.globl sqrx_382x + +.def sqrx_382x; .scl 2; .type 32; .endef +.p2align 5 +sqrx_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rsi + +.LSEH_body_sqrx_382x: + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulx_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulx_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqrx_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_382x: +.globl mulx_384 + +.def mulx_384; .scl 2; .type 32; .endef +.p2align 5 +mulx_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mulx_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +.LSEH_body_mulx_384: + + + movq %rdx,%rbx + call __mulx_384 + + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbx + + movq 40(%rsp),%rbp + + leaq 48(%rsp),%rsp + +.LSEH_epilogue_mulx_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mulx_384: + +.def __mulx_384; .scl 3; .type 32; .endef +.p2align 5 +__mulx_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + leaq -128(%rsi),%rsi + + mulxq %r14,%r9,%rcx + xorq %rbp,%rbp + + mulxq %r15,%r8,%rax + adcxq %rcx,%r8 + movq %r9,0(%rdi) + + mulxq %r10,%r9,%rcx + adcxq %rax,%r9 + + mulxq %r11,%r10,%rax + adcxq %rcx,%r10 + + mulxq %r12,%r11,%rcx + adcxq %rax,%r11 + + mulxq %r13,%r12,%r13 + movq 8(%rbx),%rdx + adcxq %rcx,%r12 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,8(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 16(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,16(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 24(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,24(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 32(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,32(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 40(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,40(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq %rax,%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + movq %r10,64(%rdi) + movq %r11,72(%rdi) + movq %r12,80(%rdi) + movq %r13,88(%rdi) + + .byte 0xf3,0xc3 + +.globl sqrx_384 + +.def sqrx_384; .scl 2; .type 32; .endef +.p2align 5 +sqrx_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_384: + movq %rcx,%rdi + movq %rdx,%rsi + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + pushq %rdi + +.LSEH_body_sqrx_384: + + + call __sqrx_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sqrx_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_384: +.def __sqrx_384; .scl 3; .type 32; .endef +.p2align 5 +__sqrx_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rdx + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%rcx + movq 32(%rsi),%rbx + + + mulxq %r14,%r8,%rdi + movq 40(%rsi),%rbp + mulxq %r15,%r9,%rax + addq %rdi,%r9 + mulxq %rcx,%r10,%rdi + adcq %rax,%r10 + mulxq %rbx,%r11,%rax + adcq %rdi,%r11 + mulxq %rbp,%r12,%r13 + movq %r14,%rdx + adcq %rax,%r12 + adcq $0,%r13 + + + xorq %r14,%r14 + mulxq %r15,%rdi,%rax + adcxq %rdi,%r10 + adoxq %rax,%r11 + + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r11 + adoxq %rax,%r12 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbp,%rdi,%rax + movq %r15,%rdx + adcxq %rdi,%r13 + adoxq %r14,%rax + adcxq %rax,%r14 + + + xorq %r15,%r15 + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r13 + adoxq %rax,%r14 + + mulxq %rbp,%rdi,%rax + movq %rcx,%rdx + adcxq %rdi,%r14 + adoxq %r15,%rax + adcxq %rax,%r15 + + + xorq %rcx,%rcx + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r14 + adoxq %rax,%r15 + + mulxq %rbp,%rdi,%rax + movq %rbx,%rdx + adcxq %rdi,%r15 + adoxq %rcx,%rax + adcxq %rax,%rcx + + + mulxq %rbp,%rdi,%rbx + movq 0(%rsi),%rdx + addq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rbx + + + xorq %rbp,%rbp + adcxq %r8,%r8 + adcxq %r9,%r9 + adcxq %r10,%r10 + adcxq %r11,%r11 + adcxq %r12,%r12 + + + mulxq %rdx,%rdx,%rax + movq %rdx,0(%rdi) + movq 8(%rsi),%rdx + adoxq %rax,%r8 + movq %r8,8(%rdi) + + mulxq %rdx,%r8,%rax + movq 16(%rsi),%rdx + adoxq %r8,%r9 + adoxq %rax,%r10 + movq %r9,16(%rdi) + movq %r10,24(%rdi) + + mulxq %rdx,%r8,%r9 + movq 24(%rsi),%rdx + adoxq %r8,%r11 + adoxq %r9,%r12 + adcxq %r13,%r13 + adcxq %r14,%r14 + movq %r11,32(%rdi) + movq %r12,40(%rdi) + + mulxq %rdx,%r8,%r9 + movq 32(%rsi),%rdx + adoxq %r8,%r13 + adoxq %r9,%r14 + adcxq %r15,%r15 + adcxq %rcx,%rcx + movq %r13,48(%rdi) + movq %r14,56(%rdi) + + mulxq %rdx,%r8,%r9 + movq 40(%rsi),%rdx + adoxq %r8,%r15 + adoxq %r9,%rcx + adcxq %rbx,%rbx + adcxq %rbp,%rbp + movq %r15,64(%rdi) + movq %rcx,72(%rdi) + + mulxq %rdx,%r8,%r9 + adoxq %r8,%rbx + adoxq %r9,%rbp + + movq %rbx,80(%rdi) + movq %rbp,88(%rdi) + + .byte 0xf3,0xc3 + + + + +.globl redcx_mont_384 + +.def redcx_mont_384; .scl 2; .type 32; .endef +.p2align 5 +redcx_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_redcx_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_redcx_mont_384: + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_redcx_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_redcx_mont_384: + + + + +.globl fromx_mont_384 + +.def fromx_mont_384; .scl 2; .type 32; .endef +.p2align 5 +fromx_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_fromx_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_fromx_mont_384: + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + + + + + movq %r14,%rax + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_fromx_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_fromx_mont_384: +.def __mulx_by_1_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__mulx_by_1_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq %rcx,%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + imulq %r8,%rdx + + + xorq %r14,%r14 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r13 + adoxq %r14,%rbp + adcxq %rbp,%r14 + imulq %r9,%rdx + + + xorq %r15,%r15 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r14 + adoxq %r15,%rbp + adcxq %rbp,%r15 + imulq %r10,%rdx + + + xorq %r8,%r8 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r8 + imulq %r11,%rdx + + + xorq %r9,%r9 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r8 + adoxq %r9,%rbp + adcxq %rbp,%r9 + imulq %r12,%rdx + + + xorq %r10,%r10 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r10 + imulq %r13,%rdx + + + xorq %r11,%r11 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r11 + .byte 0xf3,0xc3 + + +.def __redc_tail_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__redc_tail_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 + + +.globl sgn0x_pty_mont_384 + +.def sgn0x_pty_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sgn0x_pty_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0x_pty_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sgn0x_pty_mont_384: + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sgn0x_pty_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0x_pty_mont_384: + +.globl sgn0x_pty_mont_384x + +.def sgn0x_pty_mont_384x; .scl 2; .type 32; .endef +.p2align 5 +sgn0x_pty_mont_384x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sgn0x_pty_mont_384x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $8,%rsp + +.LSEH_body_sgn0x_pty_mont_384x: + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 + + movq 16(%rsp),%r14 + + movq 24(%rsp),%r13 + + movq 32(%rsp),%r12 + + movq 40(%rsp),%rbx + + movq 48(%rsp),%rbp + + leaq 56(%rsp),%rsp + +.LSEH_epilogue_sgn0x_pty_mont_384x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sgn0x_pty_mont_384x: +.globl mulx_mont_384 + +.def mulx_mont_384; .scl 2; .type 32; .endef +.p2align 5 +mulx_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_mulx_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + leaq -24(%rsp),%rsp + +.LSEH_body_mulx_mont_384: + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + movq %r8,(%rsp) + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 + + movq 32(%rsp),%r14 + + movq 40(%rsp),%r13 + + movq 48(%rsp),%r12 + + movq 56(%rsp),%rbx + + movq 64(%rsp),%rbp + + leaq 72(%rsp),%rsp + +.LSEH_epilogue_mulx_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_mulx_mont_384: +.def __mulx_mont_384; .scl 3; .type 32; .endef +.p2align 5 +__mulx_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + xorq %r15,%r15 + + movq %r8,16(%rsp) + imulq 8(%rsp),%r8 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %rbp,%r15 + adoxq %rax,%r15 + adoxq %rax,%rax + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %r8,%r14 + adoxq %r8,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r9,16(%rsp) + imulq 8(%rsp),%r9 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rbp,%rax + adoxq %r8,%rax + adoxq %r8,%r8 + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r9,%r15 + adoxq %r9,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r10,16(%rsp) + imulq 8(%rsp),%r10 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %rbp,%r8 + adoxq %r9,%r8 + adoxq %r9,%r9 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r10,%rax + adoxq %r10,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r11,16(%rsp) + imulq 8(%rsp),%r11 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %rbp,%r9 + adoxq %r10,%r9 + adoxq %r10,%r10 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r11,%r8 + adoxq %r11,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + movq %r12,16(%rsp) + imulq 8(%rsp),%r12 + + + xorq %r11,%r11 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %rbp,%r10 + adoxq %r11,%r10 + adoxq %r11,%r11 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r12,%r9 + adoxq %r12,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + movq %r15,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + movq %rax,%rsi + + mulxq 40+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + movq %r14,%rdx + adcxq %r12,%r10 + adoxq %r12,%r11 + leaq 128(%rcx),%rcx + movq %r8,%r12 + adcq $0,%r11 + + + + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r9,%rdi + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r8 + sbbq 32(%rcx),%r9 + movq %r10,%rbp + sbbq 40(%rcx),%r10 + sbbq $0,%r11 + + cmovncq %r14,%rdx + cmovcq %r13,%r15 + cmovcq %rsi,%rax + cmovncq %r8,%r12 + movq %rdx,0(%rbx) + cmovncq %r9,%rdi + movq %r15,8(%rbx) + cmovncq %r10,%rbp + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + + .byte 0xf3,0xc3 + + +.globl sqrx_mont_384 + +.def sqrx_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sqrx_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + leaq -24(%rsp),%rsp + +.LSEH_body_sqrx_mont_384: + + + movq %rcx,%r8 + leaq -128(%rdx),%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + leaq (%rsi),%rbx + movq %r8,(%rsp) + leaq -128(%rsi),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 + + movq 32(%rsp),%r14 + + movq 40(%rsp),%r13 + + movq 48(%rsp),%r12 + + movq 56(%rsp),%rbx + + movq 64(%rsp),%rbp + + leaq 72(%rsp),%rsp + +.LSEH_epilogue_sqrx_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_mont_384: + +.globl sqrx_n_mul_mont_384 + +.def sqrx_n_mul_mont_384; .scl 2; .type 32; .endef +.p2align 5 +sqrx_n_mul_mont_384: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_n_mul_mont_384: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + movq 48(%rsp),%r9 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + leaq -40(%rsp),%rsp + +.LSEH_body_sqrx_n_mul_mont_384: + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + +.Loop_sqrx_384: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movd %xmm1,%r10d + decl %r10d + jnz .Loop_sqrx_384 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 + + movq 48(%rsp),%r14 + + movq 56(%rsp),%r13 + + movq 64(%rsp),%r12 + + movq 72(%rsp),%rbx + + movq 80(%rsp),%rbp + + leaq 88(%rsp),%rsp + +.LSEH_epilogue_sqrx_n_mul_mont_384: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_n_mul_mont_384: + +.globl sqrx_n_mul_mont_383 + +.def sqrx_n_mul_mont_383; .scl 2; .type 32; .endef +.p2align 5 +sqrx_n_mul_mont_383: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_n_mul_mont_383: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + movq 40(%rsp),%r8 + movq 48(%rsp),%r9 + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + leaq -40(%rsp),%rsp + +.LSEH_body_sqrx_n_mul_mont_383: + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + leaq -128(%rcx),%rcx + +.Loop_sqrx_383: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_383_nonred + + movd %xmm1,%r10d + decl %r10d + jnz .Loop_sqrx_383 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 + + movq 48(%rsp),%r14 + + movq 56(%rsp),%r13 + + movq 64(%rsp),%r12 + + movq 72(%rsp),%rbx + + movq 80(%rsp),%rbp + + leaq 88(%rsp),%rsp + +.LSEH_epilogue_sqrx_n_mul_mont_383: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_n_mul_mont_383: +.def __mulx_mont_383_nonred; .scl 3; .type 32; .endef +.p2align 5 +__mulx_mont_383_nonred: + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + movq %r8,%rax + imulq 8(%rsp),%r8 + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %r15,%rbp + adoxq %rbp,%r15 + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %rax,%r14 + adoxq %rax,%r15 + adcxq %rax,%r15 + movq %r9,%r8 + imulq 8(%rsp),%r9 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rax,%rbp + adoxq %rbp,%rax + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r10,%r9 + imulq 8(%rsp),%r10 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %r8,%rbp + adoxq %rbp,%r8 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r11,%r10 + imulq 8(%rsp),%r11 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r9 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r12,%r11 + imulq 8(%rsp),%r12 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r10 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r14,%rdx + adcxq %rdi,%r9 + adoxq %rbp,%r10 + adcq $0,%r10 + movq %r8,%r12 + + movq %r14,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r9,%rdi + movq %r8,24(%rbx) + movq %r9,32(%rbx) + movq %r10,40(%rbx) + movq %r10,%rbp + + .byte 0xf3,0xc3 + + +.globl sqrx_mont_382x + +.def sqrx_mont_382x; .scl 2; .type 32; .endef +.p2align 5 +sqrx_mont_382x: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_sqrx_mont_382x: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + movq %r9,%rcx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $136,%rsp + +.LSEH_body_sqrx_mont_382x: + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + adcq %r12,%r12 + adcq %rdi,%rdi + adcq %rbp,%rbp + + movq %rdx,48(%rbx) + movq %r15,56(%rbx) + movq %rax,64(%rbx) + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32-128(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + + + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + movq 32+96(%rsp),%r14 + leaq 128(%rcx),%rcx + movq 32+0(%rsp),%r8 + andq %r14,%r8 + movq 32+8(%rsp),%r9 + andq %r14,%r9 + movq 32+16(%rsp),%r10 + andq %r14,%r10 + movq 32+24(%rsp),%r11 + andq %r14,%r11 + movq 32+32(%rsp),%r13 + andq %r14,%r13 + andq 32+40(%rsp),%r14 + + subq %r8,%rdx + movq 0(%rcx),%r8 + sbbq %r9,%r15 + movq 8(%rcx),%r9 + sbbq %r10,%rax + movq 16(%rcx),%r10 + sbbq %r11,%r12 + movq 24(%rcx),%r11 + sbbq %r13,%rdi + movq 32(%rcx),%r13 + sbbq %r14,%rbp + sbbq %r14,%r14 + + andq %r14,%r8 + andq %r14,%r9 + andq %r14,%r10 + andq %r14,%r11 + andq %r14,%r13 + andq 40(%rcx),%r14 + + addq %r8,%rdx + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%r12 + adcq %r13,%rdi + adcq %r14,%rbp + + movq %rdx,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 + + movq 8(%r8),%r14 + + movq 16(%r8),%r13 + + movq 24(%r8),%r12 + + movq 32(%r8),%rbx + + movq 40(%r8),%rbp + + leaq 48(%r8),%rsp + +.LSEH_epilogue_sqrx_mont_382x: + mov 8(%rsp),%rdi + mov 16(%rsp),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_sqrx_mont_382x: +.section .pdata +.p2align 2 +.rva .LSEH_begin_mulx_mont_384x +.rva .LSEH_body_mulx_mont_384x +.rva .LSEH_info_mulx_mont_384x_prologue + +.rva .LSEH_body_mulx_mont_384x +.rva .LSEH_epilogue_mulx_mont_384x +.rva .LSEH_info_mulx_mont_384x_body + +.rva .LSEH_epilogue_mulx_mont_384x +.rva .LSEH_end_mulx_mont_384x +.rva .LSEH_info_mulx_mont_384x_epilogue + +.rva .LSEH_begin_sqrx_mont_384x +.rva .LSEH_body_sqrx_mont_384x +.rva .LSEH_info_sqrx_mont_384x_prologue + +.rva .LSEH_body_sqrx_mont_384x +.rva .LSEH_epilogue_sqrx_mont_384x +.rva .LSEH_info_sqrx_mont_384x_body + +.rva .LSEH_epilogue_sqrx_mont_384x +.rva .LSEH_end_sqrx_mont_384x +.rva .LSEH_info_sqrx_mont_384x_epilogue + +.rva .LSEH_begin_mulx_382x +.rva .LSEH_body_mulx_382x +.rva .LSEH_info_mulx_382x_prologue + +.rva .LSEH_body_mulx_382x +.rva .LSEH_epilogue_mulx_382x +.rva .LSEH_info_mulx_382x_body + +.rva .LSEH_epilogue_mulx_382x +.rva .LSEH_end_mulx_382x +.rva .LSEH_info_mulx_382x_epilogue + +.rva .LSEH_begin_sqrx_382x +.rva .LSEH_body_sqrx_382x +.rva .LSEH_info_sqrx_382x_prologue + +.rva .LSEH_body_sqrx_382x +.rva .LSEH_epilogue_sqrx_382x +.rva .LSEH_info_sqrx_382x_body + +.rva .LSEH_epilogue_sqrx_382x +.rva .LSEH_end_sqrx_382x +.rva .LSEH_info_sqrx_382x_epilogue + +.rva .LSEH_begin_mulx_384 +.rva .LSEH_body_mulx_384 +.rva .LSEH_info_mulx_384_prologue + +.rva .LSEH_body_mulx_384 +.rva .LSEH_epilogue_mulx_384 +.rva .LSEH_info_mulx_384_body + +.rva .LSEH_epilogue_mulx_384 +.rva .LSEH_end_mulx_384 +.rva .LSEH_info_mulx_384_epilogue + +.rva .LSEH_begin_sqrx_384 +.rva .LSEH_body_sqrx_384 +.rva .LSEH_info_sqrx_384_prologue + +.rva .LSEH_body_sqrx_384 +.rva .LSEH_epilogue_sqrx_384 +.rva .LSEH_info_sqrx_384_body + +.rva .LSEH_epilogue_sqrx_384 +.rva .LSEH_end_sqrx_384 +.rva .LSEH_info_sqrx_384_epilogue + +.rva .LSEH_begin_redcx_mont_384 +.rva .LSEH_body_redcx_mont_384 +.rva .LSEH_info_redcx_mont_384_prologue + +.rva .LSEH_body_redcx_mont_384 +.rva .LSEH_epilogue_redcx_mont_384 +.rva .LSEH_info_redcx_mont_384_body + +.rva .LSEH_epilogue_redcx_mont_384 +.rva .LSEH_end_redcx_mont_384 +.rva .LSEH_info_redcx_mont_384_epilogue + +.rva .LSEH_begin_fromx_mont_384 +.rva .LSEH_body_fromx_mont_384 +.rva .LSEH_info_fromx_mont_384_prologue + +.rva .LSEH_body_fromx_mont_384 +.rva .LSEH_epilogue_fromx_mont_384 +.rva .LSEH_info_fromx_mont_384_body + +.rva .LSEH_epilogue_fromx_mont_384 +.rva .LSEH_end_fromx_mont_384 +.rva .LSEH_info_fromx_mont_384_epilogue + +.rva .LSEH_begin_sgn0x_pty_mont_384 +.rva .LSEH_body_sgn0x_pty_mont_384 +.rva .LSEH_info_sgn0x_pty_mont_384_prologue + +.rva .LSEH_body_sgn0x_pty_mont_384 +.rva .LSEH_epilogue_sgn0x_pty_mont_384 +.rva .LSEH_info_sgn0x_pty_mont_384_body + +.rva .LSEH_epilogue_sgn0x_pty_mont_384 +.rva .LSEH_end_sgn0x_pty_mont_384 +.rva .LSEH_info_sgn0x_pty_mont_384_epilogue + +.rva .LSEH_begin_sgn0x_pty_mont_384x +.rva .LSEH_body_sgn0x_pty_mont_384x +.rva .LSEH_info_sgn0x_pty_mont_384x_prologue + +.rva .LSEH_body_sgn0x_pty_mont_384x +.rva .LSEH_epilogue_sgn0x_pty_mont_384x +.rva .LSEH_info_sgn0x_pty_mont_384x_body + +.rva .LSEH_epilogue_sgn0x_pty_mont_384x +.rva .LSEH_end_sgn0x_pty_mont_384x +.rva .LSEH_info_sgn0x_pty_mont_384x_epilogue + +.rva .LSEH_begin_mulx_mont_384 +.rva .LSEH_body_mulx_mont_384 +.rva .LSEH_info_mulx_mont_384_prologue + +.rva .LSEH_body_mulx_mont_384 +.rva .LSEH_epilogue_mulx_mont_384 +.rva .LSEH_info_mulx_mont_384_body + +.rva .LSEH_epilogue_mulx_mont_384 +.rva .LSEH_end_mulx_mont_384 +.rva .LSEH_info_mulx_mont_384_epilogue + +.rva .LSEH_begin_sqrx_mont_384 +.rva .LSEH_body_sqrx_mont_384 +.rva .LSEH_info_sqrx_mont_384_prologue + +.rva .LSEH_body_sqrx_mont_384 +.rva .LSEH_epilogue_sqrx_mont_384 +.rva .LSEH_info_sqrx_mont_384_body + +.rva .LSEH_epilogue_sqrx_mont_384 +.rva .LSEH_end_sqrx_mont_384 +.rva .LSEH_info_sqrx_mont_384_epilogue + +.rva .LSEH_begin_sqrx_n_mul_mont_384 +.rva .LSEH_body_sqrx_n_mul_mont_384 +.rva .LSEH_info_sqrx_n_mul_mont_384_prologue + +.rva .LSEH_body_sqrx_n_mul_mont_384 +.rva .LSEH_epilogue_sqrx_n_mul_mont_384 +.rva .LSEH_info_sqrx_n_mul_mont_384_body + +.rva .LSEH_epilogue_sqrx_n_mul_mont_384 +.rva .LSEH_end_sqrx_n_mul_mont_384 +.rva .LSEH_info_sqrx_n_mul_mont_384_epilogue + +.rva .LSEH_begin_sqrx_n_mul_mont_383 +.rva .LSEH_body_sqrx_n_mul_mont_383 +.rva .LSEH_info_sqrx_n_mul_mont_383_prologue + +.rva .LSEH_body_sqrx_n_mul_mont_383 +.rva .LSEH_epilogue_sqrx_n_mul_mont_383 +.rva .LSEH_info_sqrx_n_mul_mont_383_body + +.rva .LSEH_epilogue_sqrx_n_mul_mont_383 +.rva .LSEH_end_sqrx_n_mul_mont_383 +.rva .LSEH_info_sqrx_n_mul_mont_383_epilogue + +.rva .LSEH_begin_sqrx_mont_382x +.rva .LSEH_body_sqrx_mont_382x +.rva .LSEH_info_sqrx_mont_382x_prologue + +.rva .LSEH_body_sqrx_mont_382x +.rva .LSEH_epilogue_sqrx_mont_382x +.rva .LSEH_info_sqrx_mont_382x_body + +.rva .LSEH_epilogue_sqrx_mont_382x +.rva .LSEH_end_sqrx_mont_382x +.rva .LSEH_info_sqrx_mont_382x_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_mulx_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mulx_mont_384x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x29,0x00 +.byte 0x00,0xe4,0x2a,0x00 +.byte 0x00,0xd4,0x2b,0x00 +.byte 0x00,0xc4,0x2c,0x00 +.byte 0x00,0x34,0x2d,0x00 +.byte 0x00,0x54,0x2e,0x00 +.byte 0x00,0x74,0x30,0x00 +.byte 0x00,0x64,0x31,0x00 +.byte 0x00,0x01,0x2f,0x00 +.LSEH_info_mulx_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_mont_384x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqrx_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mulx_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mulx_382x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_mulx_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_382x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqrx_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mulx_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mulx_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x00,0x00 +.byte 0x00,0xe4,0x01,0x00 +.byte 0x00,0xd4,0x02,0x00 +.byte 0x00,0xc4,0x03,0x00 +.byte 0x00,0x34,0x04,0x00 +.byte 0x00,0x54,0x05,0x00 +.byte 0x00,0x74,0x07,0x00 +.byte 0x00,0x64,0x08,0x00 +.byte 0x00,0x52 +.byte 0x00,0x00 +.LSEH_info_mulx_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sqrx_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_redcx_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_redcx_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_redcx_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_fromx_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_fromx_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_fromx_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0x_pty_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0x_pty_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sgn0x_pty_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sgn0x_pty_mont_384x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sgn0x_pty_mont_384x_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x01,0x00 +.byte 0x00,0xe4,0x02,0x00 +.byte 0x00,0xd4,0x03,0x00 +.byte 0x00,0xc4,0x04,0x00 +.byte 0x00,0x34,0x05,0x00 +.byte 0x00,0x54,0x06,0x00 +.byte 0x00,0x74,0x08,0x00 +.byte 0x00,0x64,0x09,0x00 +.byte 0x00,0x62 +.byte 0x00,0x00 +.LSEH_info_sgn0x_pty_mont_384x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_mulx_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_mulx_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x03,0x00 +.byte 0x00,0xe4,0x04,0x00 +.byte 0x00,0xd4,0x05,0x00 +.byte 0x00,0xc4,0x06,0x00 +.byte 0x00,0x34,0x07,0x00 +.byte 0x00,0x54,0x08,0x00 +.byte 0x00,0x74,0x0a,0x00 +.byte 0x00,0x64,0x0b,0x00 +.byte 0x00,0x82 +.byte 0x00,0x00 +.LSEH_info_mulx_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x03,0x00 +.byte 0x00,0xe4,0x04,0x00 +.byte 0x00,0xd4,0x05,0x00 +.byte 0x00,0xc4,0x06,0x00 +.byte 0x00,0x34,0x07,0x00 +.byte 0x00,0x54,0x08,0x00 +.byte 0x00,0x74,0x0a,0x00 +.byte 0x00,0x64,0x0b,0x00 +.byte 0x00,0x82 +.byte 0x00,0x00 +.LSEH_info_sqrx_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_n_mul_mont_384_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_n_mul_mont_384_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x05,0x00 +.byte 0x00,0xe4,0x06,0x00 +.byte 0x00,0xd4,0x07,0x00 +.byte 0x00,0xc4,0x08,0x00 +.byte 0x00,0x34,0x09,0x00 +.byte 0x00,0x54,0x0a,0x00 +.byte 0x00,0x74,0x0c,0x00 +.byte 0x00,0x64,0x0d,0x00 +.byte 0x00,0xa2 +.byte 0x00,0x00 +.LSEH_info_sqrx_n_mul_mont_384_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_n_mul_mont_383_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_n_mul_mont_383_body: +.byte 1,0,17,0 +.byte 0x00,0xf4,0x05,0x00 +.byte 0x00,0xe4,0x06,0x00 +.byte 0x00,0xd4,0x07,0x00 +.byte 0x00,0xc4,0x08,0x00 +.byte 0x00,0x34,0x09,0x00 +.byte 0x00,0x54,0x0a,0x00 +.byte 0x00,0x74,0x0c,0x00 +.byte 0x00,0x64,0x0d,0x00 +.byte 0x00,0xa2 +.byte 0x00,0x00 +.LSEH_info_sqrx_n_mul_mont_383_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + +.LSEH_info_sqrx_mont_382x_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_sqrx_mont_382x_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x11,0x00 +.byte 0x00,0xe4,0x12,0x00 +.byte 0x00,0xd4,0x13,0x00 +.byte 0x00,0xc4,0x14,0x00 +.byte 0x00,0x34,0x15,0x00 +.byte 0x00,0x54,0x16,0x00 +.byte 0x00,0x74,0x18,0x00 +.byte 0x00,0x64,0x19,0x00 +.byte 0x00,0x01,0x17,0x00 +.LSEH_info_sqrx_mont_382x_epilogue: +.byte 1,0,4,0 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x00,0x00,0x00 + diff --git a/crypto/blst_src/build/coff/sha256-armv8.S b/crypto/blst_src/build/coff/sha256-armv8.S new file mode 100644 index 00000000000..a8bcbd3631b --- /dev/null +++ b/crypto/blst_src/build/coff/sha256-armv8.S @@ -0,0 +1,1087 @@ +// +// Copyright Supranational LLC +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// ==================================================================== +// Written by Andy Polyakov, @dot-asm, initially for the OpenSSL +// project. +// ==================================================================== +// +// sha256_block procedure for ARMv8. +// +// This module is stripped of scalar code paths, with raionale that all +// known processors are NEON-capable. +// +// See original module at CRYPTOGAMS for further details. + +.text + +.p2align 6 + +.LK256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0 //terminator + +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.align 2 +.p2align 2 +.globl blst_sha256_block_armv8 +.def blst_sha256_block_armv8; +.type 32; +.endef +.p2align 6 +blst_sha256_block_armv8: +.Lv8_entry: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ld1 {v0.4s,v1.4s},[x0] + adr x3,.LK256 + +.Loop_hw: + ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 + sub x2,x2,#1 + ld1 {v16.4s},[x3],#16 + rev32 v4.16b,v4.16b + rev32 v5.16b,v5.16b + rev32 v6.16b,v6.16b + rev32 v7.16b,v7.16b + orr v18.16b,v0.16b,v0.16b // offload + orr v19.16b,v1.16b,v1.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + ld1 {v17.4s},[x3] + add v16.4s,v16.4s,v6.4s + sub x3,x3,#64*4-16 // rewind + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + add v17.4s,v17.4s,v7.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + add v0.4s,v0.4s,v18.4s + add v1.4s,v1.4s,v19.4s + + cbnz x2,.Loop_hw + + st1 {v0.4s,v1.4s},[x0] + + ldr x29,[sp],#16 + ret + +.globl blst_sha256_block_data_order +.def blst_sha256_block_data_order; +.type 32; +.endef +.p2align 4 +blst_sha256_block_data_order: + stp x29, x30, [sp, #-16]! + mov x29, sp + sub sp,sp,#16*4 + + adr x16,.LK256 + add x2,x1,x2,lsl#6 // len to point at the end of inp + + ld1 {v0.16b},[x1], #16 + ld1 {v1.16b},[x1], #16 + ld1 {v2.16b},[x1], #16 + ld1 {v3.16b},[x1], #16 + ld1 {v4.4s},[x16], #16 + ld1 {v5.4s},[x16], #16 + ld1 {v6.4s},[x16], #16 + ld1 {v7.4s},[x16], #16 + rev32 v0.16b,v0.16b // yes, even on + rev32 v1.16b,v1.16b // big-endian + rev32 v2.16b,v2.16b + rev32 v3.16b,v3.16b + mov x17,sp + add v4.4s,v4.4s,v0.4s + add v5.4s,v5.4s,v1.4s + add v6.4s,v6.4s,v2.4s + st1 {v4.4s,v5.4s},[x17], #32 + add v7.4s,v7.4s,v3.4s + st1 {v6.4s,v7.4s},[x17] + sub x17,x17,#32 + + ldp w3,w4,[x0] + ldp w5,w6,[x0,#8] + ldp w7,w8,[x0,#16] + ldp w9,w10,[x0,#24] + ldr w12,[sp,#0] + mov w13,wzr + eor w14,w4,w5 + mov w15,wzr + b .L_00_48 + +.p2align 4 +.L_00_48: + ext v4.16b,v0.16b,v1.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v2.16b,v3.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v3.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v0.4s,v0.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v0.4s,v0.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v0.4s,v0.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v0.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v0.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v0.4s,#15 + add w8,w8,w12 + ushr v17.4s,v0.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v0.4s,#13 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v0.4s,v0.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v0.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v1.16b,v2.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v3.16b,v0.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v0.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v1.4s,v1.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v1.4s,v1.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v1.4s,v1.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v1.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v1.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v1.4s,#15 + add w4,w4,w12 + ushr v17.4s,v1.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v1.4s,#13 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v1.4s,v1.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v1.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + ext v4.16b,v2.16b,v3.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v0.16b,v1.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v1.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v2.4s,v2.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v2.4s,v2.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v2.4s,v2.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v2.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v2.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v2.4s,#15 + add w8,w8,w12 + ushr v17.4s,v2.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v2.4s,#13 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v2.4s,v2.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v2.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v3.16b,v0.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v1.16b,v2.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v2.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v3.4s,v3.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v3.4s,v3.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v3.4s,v3.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v3.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v3.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v3.4s,#15 + add w4,w4,w12 + ushr v17.4s,v3.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v3.4s,#13 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v3.4s,v3.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v3.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[x16] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + cmp w12,#0 // check for K256 terminator + ldr w12,[sp,#0] + sub x17,x17,#64 + bne .L_00_48 + + sub x16,x16,#256 // rewind x16 + cmp x1,x2 + mov x17, #64 + csel x17, x17, xzr, eq + sub x1,x1,x17 // avoid SEGV + mov x17,sp + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v0.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v0.16b,v0.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v0.4s + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v1.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v1.16b,v1.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v1.4s + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v2.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v2.16b,v2.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v2.4s + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v3.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v3.16b,v3.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v3.4s + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w3,w3,w15 // h+=Sigma0(a) from the past + ldp w11,w12,[x0,#0] + add w3,w3,w13 // h+=Maj(a,b,c) from the past + ldp w13,w14,[x0,#8] + add w3,w3,w11 // accumulate + add w4,w4,w12 + ldp w11,w12,[x0,#16] + add w5,w5,w13 + add w6,w6,w14 + ldp w13,w14,[x0,#24] + add w7,w7,w11 + add w8,w8,w12 + ldr w12,[sp,#0] + stp w3,w4,[x0,#0] + add w9,w9,w13 + mov w13,wzr + stp w5,w6,[x0,#8] + add w10,w10,w14 + stp w7,w8,[x0,#16] + eor w14,w4,w5 + stp w9,w10,[x0,#24] + mov w15,wzr + mov x17,sp + b.ne .L_00_48 + + ldr x29,[x29] + add sp,sp,#16*4+16 + ret + +.globl blst_sha256_emit + +.def blst_sha256_emit; +.type 32; +.endef +.p2align 4 +blst_sha256_emit: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] +#ifndef __AARCH64EB__ + rev x4,x4 + rev x5,x5 + rev x6,x6 + rev x7,x7 +#endif + str w4,[x0,#4] + lsr x4,x4,#32 + str w5,[x0,#12] + lsr x5,x5,#32 + str w6,[x0,#20] + lsr x6,x6,#32 + str w7,[x0,#28] + lsr x7,x7,#32 + str w4,[x0,#0] + str w5,[x0,#8] + str w6,[x0,#16] + str w7,[x0,#24] + ret + + +.globl blst_sha256_bcopy + +.def blst_sha256_bcopy; +.type 32; +.endef +.p2align 4 +blst_sha256_bcopy: +.Loop_bcopy: + ldrb w3,[x1],#1 + sub x2,x2,#1 + strb w3,[x0],#1 + cbnz x2,.Loop_bcopy + ret + + +.globl blst_sha256_hcopy + +.def blst_sha256_hcopy; +.type 32; +.endef +.p2align 4 +blst_sha256_hcopy: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + stp x4,x5,[x0] + stp x6,x7,[x0,#16] + ret + diff --git a/crypto/blst_src/build/coff/sha256-portable-x86_64.s b/crypto/blst_src/build/coff/sha256-portable-x86_64.s new file mode 100644 index 00000000000..e499d107c70 --- /dev/null +++ b/crypto/blst_src/build/coff/sha256-portable-x86_64.s @@ -0,0 +1,1784 @@ +.text + +.globl blst_sha256_block_data_order +.def blst_sha256_block_data_order; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_block_data_order: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_blst_sha256_block_data_order: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbx + + pushq %rbp + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + shlq $4,%rdx + subq $64+24,%rsp + + leaq (%rsi,%rdx,4),%rdx + movq %rdi,64+0(%rsp) + movq %rsi,64+8(%rsp) + movq %rdx,64+16(%rsp) +.LSEH_body_blst_sha256_block_data_order: + + + movl 0(%rdi),%eax + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + jmp .Lloop + +.p2align 4 +.Lloop: + movl %ebx,%edi + leaq K256(%rip),%rbp + xorl %ecx,%edi + movl 0(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 0(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 4(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 4(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 8(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 8(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 12(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 12(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 16(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 16(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 20(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 20(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 24(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 24(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 28(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 28(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + addl %r14d,%eax + movl 32(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 32(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 36(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 36(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 40(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 40(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 44(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 44(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 48(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 48(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 52(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 52(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 56(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 56(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 60(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 60(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + jmp .Lrounds_16_xx +.p2align 4 +.Lrounds_16_xx: + movl 4(%rsp),%r13d + movl 56(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 36(%rsp),%r12d + + addl 0(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 64(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 8(%rsp),%r13d + movl 60(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 40(%rsp),%r12d + + addl 4(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 68(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 12(%rsp),%r13d + movl 0(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 44(%rsp),%r12d + + addl 8(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 72(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 16(%rsp),%r13d + movl 4(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 48(%rsp),%r12d + + addl 12(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 76(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 20(%rsp),%r13d + movl 8(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 52(%rsp),%r12d + + addl 16(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 80(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 24(%rsp),%r13d + movl 12(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 56(%rsp),%r12d + + addl 20(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 84(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 28(%rsp),%r13d + movl 16(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 60(%rsp),%r12d + + addl 24(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 88(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 32(%rsp),%r13d + movl 20(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 0(%rsp),%r12d + + addl 28(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 92(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + movl 36(%rsp),%r13d + movl 24(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 4(%rsp),%r12d + + addl 32(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 96(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 40(%rsp),%r13d + movl 28(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 8(%rsp),%r12d + + addl 36(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 100(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 44(%rsp),%r13d + movl 32(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 12(%rsp),%r12d + + addl 40(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 104(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 48(%rsp),%r13d + movl 36(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 16(%rsp),%r12d + + addl 44(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 108(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 52(%rsp),%r13d + movl 40(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 20(%rsp),%r12d + + addl 48(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 112(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 56(%rsp),%r13d + movl 44(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 24(%rsp),%r12d + + addl 52(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 116(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 60(%rsp),%r13d + movl 48(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 28(%rsp),%r12d + + addl 56(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 120(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 0(%rsp),%r13d + movl 52(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 32(%rsp),%r12d + + addl 60(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 124(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + leaq 64(%rbp),%rbp + cmpb $0x19,3(%rbp) + jnz .Lrounds_16_xx + + movq 64+0(%rsp),%rdi + addl %r14d,%eax + leaq 64(%rsi),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + cmpq 64+16(%rsp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb .Lloop + + leaq 64+24+48(%rsp),%r11 + + movq 64+24(%rsp),%r15 + + movq -40(%r11),%r14 + + movq -32(%r11),%r13 + + movq -24(%r11),%r12 + + movq -16(%r11),%rbp + + movq -8(%r11),%rbx + +.LSEH_epilogue_blst_sha256_block_data_order: + mov 8(%r11),%rdi + mov 16(%r11),%rsi + + leaq (%r11),%rsp + .byte 0xf3,0xc3 + +.LSEH_end_blst_sha256_block_data_order: + +.p2align 6 + +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl blst_sha256_emit + +.def blst_sha256_emit; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_emit: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + bswapq %r8 + movq 24(%rdx),%r11 + bswapq %r9 + movl %r8d,4(%rcx) + bswapq %r10 + movl %r9d,12(%rcx) + bswapq %r11 + movl %r10d,20(%rcx) + shrq $32,%r8 + movl %r11d,28(%rcx) + shrq $32,%r9 + movl %r8d,0(%rcx) + shrq $32,%r10 + movl %r9d,8(%rcx) + shrq $32,%r11 + movl %r10d,16(%rcx) + movl %r11d,24(%rcx) + .byte 0xf3,0xc3 + + +.globl blst_sha256_bcopy + +.def blst_sha256_bcopy; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_bcopy: + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rdx,%rcx +.Loop_bcopy: + movzbl (%rdx),%eax + leaq 1(%rdx),%rdx + movb %al,-1(%rcx,%rdx,1) + decq %r8 + jnz .Loop_bcopy + .byte 0xf3,0xc3 + + +.globl blst_sha256_hcopy + +.def blst_sha256_hcopy; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_hcopy: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq %r8,0(%rcx) + movq %r9,8(%rcx) + movq %r10,16(%rcx) + movq %r11,24(%rcx) + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_blst_sha256_block_data_order +.rva .LSEH_body_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_prologue + +.rva .LSEH_body_blst_sha256_block_data_order +.rva .LSEH_epilogue_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_body + +.rva .LSEH_epilogue_blst_sha256_block_data_order +.rva .LSEH_end_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_blst_sha256_block_data_order_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_blst_sha256_block_data_order_body: +.byte 1,0,18,0 +.byte 0x00,0xf4,0x0b,0x00 +.byte 0x00,0xe4,0x0c,0x00 +.byte 0x00,0xd4,0x0d,0x00 +.byte 0x00,0xc4,0x0e,0x00 +.byte 0x00,0x54,0x0f,0x00 +.byte 0x00,0x34,0x10,0x00 +.byte 0x00,0x74,0x12,0x00 +.byte 0x00,0x64,0x13,0x00 +.byte 0x00,0x01,0x11,0x00 +.LSEH_info_blst_sha256_block_data_order_epilogue: +.byte 1,0,5,11 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x03 +.byte 0x00,0x00 + diff --git a/crypto/blst_src/build/coff/sha256-x86_64.s b/crypto/blst_src/build/coff/sha256-x86_64.s new file mode 100644 index 00000000000..ed28b781d4c --- /dev/null +++ b/crypto/blst_src/build/coff/sha256-x86_64.s @@ -0,0 +1,1560 @@ +.text + +.p2align 6 + +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f +.long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff +.long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl blst_sha256_block_data_order_shaext + +.def blst_sha256_block_data_order_shaext; .scl 2; .type 32; .endef +.p2align 6 +blst_sha256_block_data_order_shaext: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_blst_sha256_block_data_order_shaext: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + subq $0x58,%rsp + + movaps %xmm6,-88(%r11) + + movaps %xmm7,-72(%r11) + + movaps %xmm8,-56(%r11) + + movaps %xmm9,-40(%r11) + + movaps %xmm10,-24(%r11) + +.LSEH_body_blst_sha256_block_data_order_shaext: + + leaq K256+128(%rip),%rcx + movdqu (%rdi),%xmm1 + movdqu 16(%rdi),%xmm2 + movdqa 256-128(%rcx),%xmm7 + + pshufd $0x1b,%xmm1,%xmm0 + pshufd $0xb1,%xmm1,%xmm1 + pshufd $0x1b,%xmm2,%xmm2 + movdqa %xmm7,%xmm8 +.byte 102,15,58,15,202,8 + punpcklqdq %xmm0,%xmm2 + jmp .Loop_shaext + +.p2align 4 +.Loop_shaext: + movdqu (%rsi),%xmm3 + movdqu 16(%rsi),%xmm4 + movdqu 32(%rsi),%xmm5 +.byte 102,15,56,0,223 + movdqu 48(%rsi),%xmm6 + + movdqa 0-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 102,15,56,0,231 + movdqa %xmm2,%xmm10 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + nop + movdqa %xmm1,%xmm9 +.byte 15,56,203,202 + + movdqa 16-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 102,15,56,0,239 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + leaq 64(%rsi),%rsi +.byte 15,56,204,220 +.byte 15,56,203,202 + + movdqa 32-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 102,15,56,0,247 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + + movdqa 48-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 64-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 80-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 96-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 112-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 128-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 144-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 160-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 176-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 192-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 208-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 +.byte 15,56,203,202 + paddd %xmm7,%xmm6 + + movdqa 224-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 +.byte 15,56,205,245 + movdqa %xmm8,%xmm7 +.byte 15,56,203,202 + + movdqa 240-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 + nop +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + decq %rdx + nop +.byte 15,56,203,202 + + paddd %xmm10,%xmm2 + paddd %xmm9,%xmm1 + jnz .Loop_shaext + + pshufd $0xb1,%xmm2,%xmm2 + pshufd $0x1b,%xmm1,%xmm7 + pshufd $0xb1,%xmm1,%xmm1 + punpckhqdq %xmm2,%xmm1 +.byte 102,15,58,15,215,8 + + movdqu %xmm1,(%rdi) + movdqu %xmm2,16(%rdi) + movaps -88(%r11),%xmm6 + movaps -72(%r11),%xmm7 + movaps -56(%r11),%xmm8 + movaps -40(%r11),%xmm9 + movaps -24(%r11),%xmm10 + movq %r11,%rsp + +.LSEH_epilogue_blst_sha256_block_data_order_shaext: + mov 8(%r11),%rdi + mov 16(%r11),%rsi + + .byte 0xf3,0xc3 + +.LSEH_end_blst_sha256_block_data_order_shaext: +.globl blst_sha256_block_data_order + +.def blst_sha256_block_data_order; .scl 2; .type 32; .endef +.p2align 6 +blst_sha256_block_data_order: + .byte 0xf3,0x0f,0x1e,0xfa + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + movq %rsp,%r11 +.LSEH_begin_blst_sha256_block_data_order: + movq %rcx,%rdi + movq %rdx,%rsi + movq %r8,%rdx + + + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + shlq $4,%rdx + subq $104,%rsp + + leaq (%rsi,%rdx,4),%rdx + movq %rdi,0(%rsp) + + movq %rdx,16(%rsp) + movaps %xmm6,32(%rsp) + + movaps %xmm7,48(%rsp) + + movaps %xmm8,64(%rsp) + + movaps %xmm9,80(%rsp) + + movq %rsp,%rbp + +.LSEH_body_blst_sha256_block_data_order: + + + leaq -64(%rsp),%rsp + movl 0(%rdi),%eax + andq $-64,%rsp + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + + + jmp .Lloop_ssse3 +.p2align 4 +.Lloop_ssse3: + movdqa K256+256(%rip),%xmm7 + movq %rsi,8(%rbp) + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 +.byte 102,15,56,0,199 + movdqu 48(%rsi),%xmm3 + leaq K256(%rip),%rsi +.byte 102,15,56,0,207 + movdqa 0(%rsi),%xmm4 + movdqa 16(%rsi),%xmm5 +.byte 102,15,56,0,215 + paddd %xmm0,%xmm4 + movdqa 32(%rsi),%xmm6 +.byte 102,15,56,0,223 + movdqa 48(%rsi),%xmm7 + paddd %xmm1,%xmm5 + paddd %xmm2,%xmm6 + paddd %xmm3,%xmm7 + movdqa %xmm4,0(%rsp) + movl %eax,%r14d + movdqa %xmm5,16(%rsp) + movl %ebx,%edi + movdqa %xmm6,32(%rsp) + xorl %ecx,%edi + movdqa %xmm7,48(%rsp) + movl %r8d,%r13d + jmp .Lssse3_00_47 + +.p2align 4 +.Lssse3_00_47: + subq $-64,%rsi + rorl $14,%r13d + movdqa %xmm1,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm3,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,224,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,250,4 + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm3,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 4(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm0 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm0 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm0,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 0(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm0,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,0(%rsp) + rorl $14,%r13d + movdqa %xmm2,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm0,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,225,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,251,4 + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm0,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 20(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm1 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm1 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm1,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 16(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm1,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,16(%rsp) + rorl $14,%r13d + movdqa %xmm3,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm1,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,226,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,248,4 + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm1,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 36(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm2 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm2 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm2,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 32(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm2,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,32(%rsp) + rorl $14,%r13d + movdqa %xmm0,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm2,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,227,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,249,4 + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm2,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 52(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm3 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm3 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm3,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 48(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm3,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,48(%rsp) + cmpb $0,67(%rsi) + jne .Lssse3_00_47 + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 4(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 20(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 36(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 52(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + movq 0(%rbp),%rdi + movl %r14d,%eax + movq 8(%rbp),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + leaq 64(%rsi),%rsi + cmpq 16(%rbp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb .Lloop_ssse3 + + xorps %xmm0,%xmm0 + leaq 104+48(%rbp),%r11 + + movaps %xmm0,0(%rsp) + movaps %xmm0,16(%rsp) + movaps %xmm0,32(%rsp) + movaps %xmm0,48(%rsp) + movaps 32(%rbp),%xmm6 + movaps 48(%rbp),%xmm7 + movaps 64(%rbp),%xmm8 + movaps 80(%rbp),%xmm9 + movq 104(%rbp),%r15 + + movq -40(%r11),%r14 + + movq -32(%r11),%r13 + + movq -24(%r11),%r12 + + movq -16(%r11),%rbx + + movq -8(%r11),%rbp + +.LSEH_epilogue_blst_sha256_block_data_order: + mov 8(%r11),%rdi + mov 16(%r11),%rsi + + leaq (%r11),%rsp + .byte 0xf3,0xc3 + +.LSEH_end_blst_sha256_block_data_order: +.globl blst_sha256_emit + +.def blst_sha256_emit; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_emit: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + bswapq %r8 + movq 24(%rdx),%r11 + bswapq %r9 + movl %r8d,4(%rcx) + bswapq %r10 + movl %r9d,12(%rcx) + bswapq %r11 + movl %r10d,20(%rcx) + shrq $32,%r8 + movl %r11d,28(%rcx) + shrq $32,%r9 + movl %r8d,0(%rcx) + shrq $32,%r10 + movl %r9d,8(%rcx) + shrq $32,%r11 + movl %r10d,16(%rcx) + movl %r11d,24(%rcx) + .byte 0xf3,0xc3 + + +.globl blst_sha256_bcopy + +.def blst_sha256_bcopy; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_bcopy: + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rdx,%rcx +.Loop_bcopy: + movzbl (%rdx),%eax + leaq 1(%rdx),%rdx + movb %al,-1(%rcx,%rdx,1) + decq %r8 + jnz .Loop_bcopy + .byte 0xf3,0xc3 + + +.globl blst_sha256_hcopy + +.def blst_sha256_hcopy; .scl 2; .type 32; .endef +.p2align 4 +blst_sha256_hcopy: + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq %r8,0(%rcx) + movq %r9,8(%rcx) + movq %r10,16(%rcx) + movq %r11,24(%rcx) + .byte 0xf3,0xc3 + +.section .pdata +.p2align 2 +.rva .LSEH_begin_blst_sha256_block_data_order_shaext +.rva .LSEH_body_blst_sha256_block_data_order_shaext +.rva .LSEH_info_blst_sha256_block_data_order_shaext_prologue + +.rva .LSEH_body_blst_sha256_block_data_order_shaext +.rva .LSEH_epilogue_blst_sha256_block_data_order_shaext +.rva .LSEH_info_blst_sha256_block_data_order_shaext_body + +.rva .LSEH_epilogue_blst_sha256_block_data_order_shaext +.rva .LSEH_end_blst_sha256_block_data_order_shaext +.rva .LSEH_info_blst_sha256_block_data_order_shaext_epilogue + +.rva .LSEH_begin_blst_sha256_block_data_order +.rva .LSEH_body_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_prologue + +.rva .LSEH_body_blst_sha256_block_data_order +.rva .LSEH_epilogue_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_body + +.rva .LSEH_epilogue_blst_sha256_block_data_order +.rva .LSEH_end_blst_sha256_block_data_order +.rva .LSEH_info_blst_sha256_block_data_order_epilogue + +.section .xdata +.p2align 3 +.LSEH_info_blst_sha256_block_data_order_shaext_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_blst_sha256_block_data_order_shaext_body: +.byte 1,0,15,0 +.byte 0x00,0x68,0x00,0x00 +.byte 0x00,0x78,0x01,0x00 +.byte 0x00,0x88,0x02,0x00 +.byte 0x00,0x98,0x03,0x00 +.byte 0x00,0xa8,0x04,0x00 +.byte 0x00,0x74,0x0c,0x00 +.byte 0x00,0x64,0x0d,0x00 +.byte 0x00,0xa2 +.byte 0x00,0x00,0x00,0x00,0x00,0x00 +.LSEH_info_blst_sha256_block_data_order_shaext_epilogue: +.byte 1,0,5,11 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x03 +.byte 0x00,0x00 + +.LSEH_info_blst_sha256_block_data_order_prologue: +.byte 1,0,5,0x0b +.byte 0,0x74,1,0 +.byte 0,0x64,2,0 +.byte 0,0x03 +.byte 0,0 +.LSEH_info_blst_sha256_block_data_order_body: +.byte 1,0,26,5 +.byte 0x00,0x68,0x02,0x00 +.byte 0x00,0x78,0x03,0x00 +.byte 0x00,0x88,0x04,0x00 +.byte 0x00,0x98,0x05,0x00 +.byte 0x00,0xf4,0x0d,0x00 +.byte 0x00,0xe4,0x0e,0x00 +.byte 0x00,0xd4,0x0f,0x00 +.byte 0x00,0xc4,0x10,0x00 +.byte 0x00,0x34,0x11,0x00 +.byte 0x00,0x74,0x14,0x00 +.byte 0x00,0x64,0x15,0x00 +.byte 0x00,0x03 +.byte 0x00,0x01,0x12,0x00 +.byte 0x00,0x50 +.LSEH_info_blst_sha256_block_data_order_epilogue: +.byte 1,0,5,11 +.byte 0x00,0x74,0x01,0x00 +.byte 0x00,0x64,0x02,0x00 +.byte 0x00,0x03 +.byte 0x00,0x00 + diff --git a/crypto/blst_src/build/elf/add_mod_256-armv8.S b/crypto/blst_src/build/elf/add_mod_256-armv8.S new file mode 100644 index 00000000000..57476aaa1da --- /dev/null +++ b/crypto/blst_src/build/elf/add_mod_256-armv8.S @@ -0,0 +1,379 @@ +.text + +.globl add_mod_256 +.hidden add_mod_256 +.type add_mod_256,%function +.align 5 +add_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + adds x8,x8,x12 + ldp x14,x15,[x2,#16] + adcs x9,x9,x13 + ldp x4,x5,[x3] + adcs x10,x10,x14 + ldp x6,x7,[x3,#16] + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret +.size add_mod_256,.-add_mod_256 + +.globl mul_by_3_mod_256 +.hidden mul_by_3_mod_256 +.type mul_by_3_mod_256,%function +.align 5 +mul_by_3_mod_256: + ldp x12,x13,[x1] + ldp x14,x15,[x1,#16] + + adds x8,x12,x12 + ldp x4,x5,[x2] + adcs x9,x13,x13 + ldp x6,x7,[x2,#16] + adcs x10,x14,x14 + adcs x11,x15,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + adds x8,x8,x12 + adcs x9,x9,x13 + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret +.size mul_by_3_mod_256,.-mul_by_3_mod_256 + +.globl lshift_mod_256 +.hidden lshift_mod_256 +.type lshift_mod_256,%function +.align 5 +lshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +.Loop_lshift_mod_256: + adds x8,x8,x8 + sub x2,x2,#1 + adcs x9,x9,x9 + adcs x10,x10,x10 + adcs x11,x11,x11 + adc x3,xzr,xzr + + subs x12,x8,x4 + sbcs x13,x9,x5 + sbcs x14,x10,x6 + sbcs x15,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x12,lo + csel x9,x9,x13,lo + csel x10,x10,x14,lo + csel x11,x11,x15,lo + + cbnz x2,.Loop_lshift_mod_256 + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret +.size lshift_mod_256,.-lshift_mod_256 + +.globl rshift_mod_256 +.hidden rshift_mod_256 +.type rshift_mod_256,%function +.align 5 +rshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +.Loop_rshift: + adds x12,x8,x4 + sub x2,x2,#1 + adcs x13,x9,x5 + adcs x14,x10,x6 + adcs x15,x11,x7 + adc x3,xzr,xzr + tst x8,#1 + + csel x12,x12,x8,ne + csel x13,x13,x9,ne + csel x14,x14,x10,ne + csel x15,x15,x11,ne + csel x3,x3,xzr,ne + + extr x8,x13,x12,#1 + extr x9,x14,x13,#1 + extr x10,x15,x14,#1 + extr x11,x3,x15,#1 + + cbnz x2,.Loop_rshift + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret +.size rshift_mod_256,.-rshift_mod_256 + +.globl cneg_mod_256 +.hidden cneg_mod_256 +.type cneg_mod_256,%function +.align 5 +cneg_mod_256: + ldp x8,x9,[x1] + ldp x4,x5,[x3] + + ldp x10,x11,[x1,#16] + subs x12,x4,x8 + ldp x6,x7,[x3,#16] + orr x4,x8,x9 + sbcs x13,x5,x9 + orr x5,x10,x11 + sbcs x14,x6,x10 + orr x3,x4,x5 + sbc x15,x7,x11 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x8,x8,x12,eq + csel x9,x9,x13,eq + csel x10,x10,x14,eq + stp x8,x9,[x0] + csel x11,x11,x15,eq + stp x10,x11,[x0,#16] + + ret +.size cneg_mod_256,.-cneg_mod_256 + +.globl sub_mod_256 +.hidden sub_mod_256 +.type sub_mod_256,%function +.align 5 +sub_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + subs x8,x8,x12 + ldp x14,x15,[x2,#16] + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + stp x8,x9,[x0] + adc x11,x11,x7 + stp x10,x11,[x0,#16] + + ret +.size sub_mod_256,.-sub_mod_256 + +.globl check_mod_256 +.hidden check_mod_256 +.type check_mod_256,%function +.align 5 +check_mod_256: + ldp x8,x9,[x0] + ldp x10,x11,[x0,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + subs xzr,x8,x4 + sbcs xzr,x9,x5 + orr x8,x8,x9 + sbcs xzr,x10,x6 + orr x8,x8,x10 + sbcs xzr,x11,x7 + orr x8,x8,x11 + sbc x1,xzr,xzr + + cmp x8,#0 + mov x0,#1 + csel x0,x0,xzr,ne + and x0,x0,x1 + + ret +.size check_mod_256,.-check_mod_256 + +.globl add_n_check_mod_256 +.hidden add_n_check_mod_256 +.type add_n_check_mod_256,%function +.align 5 +add_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + adds x8,x8,x12 + ldp x4,x5,[x3] + adcs x9,x9,x13 + ldp x6,x7,[x3,#16] + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret +.size add_n_check_mod_256,.-add_n_check_mod_256 + +.globl sub_n_check_mod_256 +.hidden sub_n_check_mod_256 +.type sub_n_check_mod_256,%function +.align 5 +sub_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + subs x8,x8,x12 + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + adc x11,x11,x7 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret +.size sub_n_check_mod_256,.-sub_n_check_mod_256 diff --git a/crypto/blst_src/build/elf/add_mod_256-x86_64.s b/crypto/blst_src/build/elf/add_mod_256-x86_64.s new file mode 100644 index 00000000000..2f41781959c --- /dev/null +++ b/crypto/blst_src/build/elf/add_mod_256-x86_64.s @@ -0,0 +1,572 @@ +.text + +.globl add_mod_256 +.hidden add_mod_256 +.type add_mod_256,@function +.align 32 +add_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loaded_a_add_mod_256: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size add_mod_256,.-add_mod_256 + + +.globl mul_by_3_mod_256 +.hidden mul_by_3_mod_256 +.type mul_by_3_mod_256,@function +.align 32 +mul_by_3_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq %rdx,%rcx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rsi,%rdx + movq 24(%rsi),%r11 + + call __lshift_mod_256 + movq 0(%rsp),%r12 +.cfi_restore %r12 + jmp .Loaded_a_add_mod_256 + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_3_mod_256,.-mul_by_3_mod_256 + +.type __lshift_mod_256,@function +.align 32 +__lshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + movq %r8,%rax + adcq %r10,%r10 + movq %r9,%rsi + adcq %r11,%r11 + sbbq %r12,%r12 + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + cmovcq %rbx,%r10 + cmovcq %rbp,%r11 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __lshift_mod_256,.-__lshift_mod_256 + + +.globl lshift_mod_256 +.hidden lshift_mod_256 +.type lshift_mod_256,@function +.align 32 +lshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loop_lshift_mod_256: + call __lshift_mod_256 + decl %edx + jnz .Loop_lshift_mod_256 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size lshift_mod_256,.-lshift_mod_256 + + +.globl rshift_mod_256 +.hidden rshift_mod_256 +.type rshift_mod_256,@function +.align 32 +rshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rbp + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +.Loop_rshift_mod_256: + movq %rbp,%r8 + andq $1,%rbp + movq 0(%rcx),%rax + negq %rbp + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + + andq %rbp,%rax + andq %rbp,%rsi + andq %rbp,%rbx + andq 24(%rcx),%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + adcq %rbx,%r10 + adcq %rbp,%r11 + sbbq %rax,%rax + + shrq $1,%r8 + movq %r9,%rbp + shrq $1,%r9 + movq %r10,%rbx + shrq $1,%r10 + movq %r11,%rsi + shrq $1,%r11 + + shlq $63,%rbp + shlq $63,%rbx + orq %r8,%rbp + shlq $63,%rsi + orq %rbx,%r9 + shlq $63,%rax + orq %rsi,%r10 + orq %rax,%r11 + + decl %edx + jnz .Loop_rshift_mod_256 + + movq %rbp,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size rshift_mod_256,.-rshift_mod_256 + + +.globl cneg_mod_256 +.hidden cneg_mod_256 +.type cneg_mod_256,@function +.align 32 +cneg_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq 0(%rsi),%r12 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %r12,%r8 + movq 24(%rsi),%r11 + orq %r9,%r12 + orq %r10,%r12 + orq %r11,%r12 + movq $-1,%rbp + + movq 0(%rcx),%rax + cmovnzq %rbp,%r12 + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + andq %r12,%rax + movq 24(%rcx),%rbp + andq %r12,%rsi + andq %r12,%rbx + andq %r12,%rbp + + subq %r8,%rax + sbbq %r9,%rsi + sbbq %r10,%rbx + sbbq %r11,%rbp + + orq %rdx,%rdx + + cmovzq %r8,%rax + cmovzq %r9,%rsi + movq %rax,0(%rdi) + cmovzq %r10,%rbx + movq %rsi,8(%rdi) + cmovzq %r11,%rbp + movq %rbx,16(%rdi) + movq %rbp,24(%rdi) + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size cneg_mod_256,.-cneg_mod_256 + + +.globl sub_mod_256 +.hidden sub_mod_256 +.type sub_mod_256,@function +.align 32 +sub_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sub_mod_256,.-sub_mod_256 + + +.globl check_mod_256 +.hidden check_mod_256 +.type check_mod_256,@function +.align 32 +check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + movq 0(%rdi),%rax + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + + movq %rax,%r8 + orq %r9,%rax + orq %r10,%rax + orq %r11,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq %rsi,%rsi + + movq $1,%rdx + cmpq $0,%rax + cmovneq %rdx,%rax + andq %rsi,%rax + + .byte 0xf3,0xc3 +.cfi_endproc +.size check_mod_256,.-check_mod_256 + + +.globl add_n_check_mod_256 +.hidden add_n_check_mod_256 +.type add_n_check_mod_256,@function +.align 32 +add_n_check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size add_n_check_mod_256,.-add_n_check_mod_256 + + +.globl sub_n_check_mod_256 +.hidden sub_n_check_mod_256 +.type sub_n_check_mod_256,@function +.align 32 +sub_n_check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sub_n_check_mod_256,.-sub_n_check_mod_256 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/add_mod_384-armv8.S b/crypto/blst_src/build/elf/add_mod_384-armv8.S new file mode 100644 index 00000000000..5c18d7fe892 --- /dev/null +++ b/crypto/blst_src/build/elf/add_mod_384-armv8.S @@ -0,0 +1,1000 @@ +.text + +.globl add_mod_384 +.hidden add_mod_384 +.type add_mod_384,%function +.align 5 +add_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size add_mod_384,.-add_mod_384 + +.type __add_mod_384,%function +.align 5 +__add_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + +__add_mod_384_ab_are_loaded: + adds x10,x10,x16 + adcs x11,x11,x17 + adcs x12,x12,x19 + adcs x13,x13,x20 + adcs x14,x14,x21 + adcs x15,x15,x22 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret +.size __add_mod_384,.-__add_mod_384 + +.globl add_mod_384x +.hidden add_mod_384x +.type add_mod_384x,%function +.align 5 +add_mod_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size add_mod_384x,.-add_mod_384x + +.globl rshift_mod_384 +.hidden rshift_mod_384 +.type rshift_mod_384,%function +.align 5 +rshift_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +.Loop_rshift_mod_384: + sub x2,x2,#1 + bl __rshift_mod_384 + cbnz x2,.Loop_rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size rshift_mod_384,.-rshift_mod_384 + +.type __rshift_mod_384,%function +.align 5 +__rshift_mod_384: + sbfx x22,x10,#0,#1 + and x16,x22,x4 + and x17,x22,x5 + adds x10,x10,x16 + and x19,x22,x6 + adcs x11,x11,x17 + and x20,x22,x7 + adcs x12,x12,x19 + and x21,x22,x8 + adcs x13,x13,x20 + and x22,x22,x9 + adcs x14,x14,x21 + extr x10,x11,x10,#1 // a[0:5] >>= 1 + adcs x15,x15,x22 + extr x11,x12,x11,#1 + adc x22,xzr,xzr + extr x12,x13,x12,#1 + extr x13,x14,x13,#1 + extr x14,x15,x14,#1 + extr x15,x22,x15,#1 + ret +.size __rshift_mod_384,.-__rshift_mod_384 + +.globl div_by_2_mod_384 +.hidden div_by_2_mod_384 +.type div_by_2_mod_384,%function +.align 5 +div_by_2_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size div_by_2_mod_384,.-div_by_2_mod_384 + +.globl lshift_mod_384 +.hidden lshift_mod_384 +.type lshift_mod_384,%function +.align 5 +lshift_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +.Loop_lshift_mod_384: + sub x2,x2,#1 + bl __lshift_mod_384 + cbnz x2,.Loop_lshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size lshift_mod_384,.-lshift_mod_384 + +.type __lshift_mod_384,%function +.align 5 +__lshift_mod_384: + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret +.size __lshift_mod_384,.-__lshift_mod_384 + +.globl mul_by_3_mod_384 +.hidden mul_by_3_mod_384 +.type mul_by_3_mod_384,%function +.align 5 +mul_by_3_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size mul_by_3_mod_384,.-mul_by_3_mod_384 + +.globl mul_by_8_mod_384 +.hidden mul_by_8_mod_384 +.type mul_by_8_mod_384,%function +.align 5 +mul_by_8_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size mul_by_8_mod_384,.-mul_by_8_mod_384 + +.globl mul_by_3_mod_384x +.hidden mul_by_3_mod_384x +.type mul_by_3_mod_384x,%function +.align 5 +mul_by_3_mod_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + + ldp x16,x17,[x1,#48] + ldp x19,x20,[x1,#64] + ldp x21,x22,[x1,#80] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size mul_by_3_mod_384x,.-mul_by_3_mod_384x + +.globl mul_by_8_mod_384x +.hidden mul_by_8_mod_384x +.type mul_by_8_mod_384x,%function +.align 5 +mul_by_8_mod_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size mul_by_8_mod_384x,.-mul_by_8_mod_384x + +.globl cneg_mod_384 +.hidden cneg_mod_384 +.type cneg_mod_384,%function +.align 5 +cneg_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x4,x5,[x3] + ldp x12,x13,[x1,#16] + ldp x6,x7,[x3,#16] + + subs x16,x4,x10 + ldp x14,x15,[x1,#32] + ldp x8,x9,[x3,#32] + orr x3,x10,x11 + sbcs x17,x5,x11 + orr x3,x3,x12 + sbcs x19,x6,x12 + orr x3,x3,x13 + sbcs x20,x7,x13 + orr x3,x3,x14 + sbcs x21,x8,x14 + orr x3,x3,x15 + sbc x22,x9,x15 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x10,x10,x16,eq + csel x11,x11,x17,eq + csel x12,x12,x19,eq + csel x13,x13,x20,eq + stp x10,x11,[x0] + csel x14,x14,x21,eq + stp x12,x13,[x0,#16] + csel x15,x15,x22,eq + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size cneg_mod_384,.-cneg_mod_384 + +.globl sub_mod_384 +.hidden sub_mod_384 +.type sub_mod_384,%function +.align 5 +sub_mod_384: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size sub_mod_384,.-sub_mod_384 + +.type __sub_mod_384,%function +.align 5 +__sub_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + + subs x10,x10,x16 + sbcs x11,x11,x17 + sbcs x12,x12,x19 + sbcs x13,x13,x20 + sbcs x14,x14,x21 + sbcs x15,x15,x22 + sbc x3,xzr,xzr + + and x16,x4,x3 + and x17,x5,x3 + adds x10,x10,x16 + and x19,x6,x3 + adcs x11,x11,x17 + and x20,x7,x3 + adcs x12,x12,x19 + and x21,x8,x3 + adcs x13,x13,x20 + and x22,x9,x3 + adcs x14,x14,x21 + adc x15,x15,x22 + + ret +.size __sub_mod_384,.-__sub_mod_384 + +.globl sub_mod_384x +.hidden sub_mod_384x +.type sub_mod_384x,%function +.align 5 +sub_mod_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size sub_mod_384x,.-sub_mod_384x + +.globl mul_by_1_plus_i_mod_384x +.hidden mul_by_1_plus_i_mod_384x +.type mul_by_1_plus_i_mod_384x,%function +.align 5 +mul_by_1_plus_i_mod_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + add x2,x1,#48 + + bl __sub_mod_384 // a->re - a->im + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __add_mod_384_ab_are_loaded // a->re + a->im + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x + +.globl sgn0_pty_mod_384 +.hidden sgn0_pty_mod_384 +.type sgn0_pty_mod_384,%function +.align 5 +sgn0_pty_mod_384: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x0,x10,#1 + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x3,x3,xzr + + mvn x3,x3 + and x3,x3,#2 + orr x0,x0,x3 + + ret +.size sgn0_pty_mod_384,.-sgn0_pty_mod_384 + +.globl sgn0_pty_mod_384x +.hidden sgn0_pty_mod_384x +.type sgn0_pty_mod_384x,%function +.align 5 +sgn0_pty_mod_384x: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x2,x10,#1 + orr x3,x10,x11 + adds x10,x10,x10 + orr x3,x3,x12 + adcs x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + ldp x10,x11,[x0,#48] + ldp x12,x13,[x0,#64] + ldp x14,x15,[x0,#80] + + mvn x16,x16 + and x16,x16,#2 + orr x2,x2,x16 + + and x0,x10,#1 + orr x1,x10,x11 + adds x10,x10,x10 + orr x1,x1,x12 + adcs x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + mvn x16,x16 + and x16,x16,#2 + orr x0,x0,x16 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ret +.size sgn0_pty_mod_384x,.-sgn0_pty_mod_384x +.globl vec_select_32 +.hidden vec_select_32 +.type vec_select_32,%function +.align 5 +vec_select_32: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret +.size vec_select_32,.-vec_select_32 +.globl vec_select_48 +.hidden vec_select_48 +.type vec_select_48,%function +.align 5 +vec_select_48: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret +.size vec_select_48,.-vec_select_48 +.globl vec_select_96 +.hidden vec_select_96 +.type vec_select_96,%function +.align 5 +vec_select_96: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret +.size vec_select_96,.-vec_select_96 +.globl vec_select_192 +.hidden vec_select_192 +.type vec_select_192,%function +.align 5 +vec_select_192: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret +.size vec_select_192,.-vec_select_192 +.globl vec_select_144 +.hidden vec_select_144 +.type vec_select_144,%function +.align 5 +vec_select_144: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret +.size vec_select_144,.-vec_select_144 +.globl vec_select_288 +.hidden vec_select_288 +.type vec_select_288,%function +.align 5 +vec_select_288: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret +.size vec_select_288,.-vec_select_288 +.globl vec_prefetch +.hidden vec_prefetch +.type vec_prefetch,%function +.align 5 +vec_prefetch: + add x1, x1, x0 + sub x1, x1, #1 + mov x2, #64 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + prfm pldl1keep, [x0] + ret +.size vec_prefetch,.-vec_prefetch +.globl vec_is_zero_16x +.hidden vec_is_zero_16x +.type vec_is_zero_16x,%function +.align 5 +vec_is_zero_16x: + ld1 {v0.2d}, [x0], #16 + lsr x1, x1, #4 + sub x1, x1, #1 + cbz x1, .Loop_is_zero_done + +.Loop_is_zero: + ld1 {v1.2d}, [x0], #16 + orr v0.16b, v0.16b, v1.16b + sub x1, x1, #1 + cbnz x1, .Loop_is_zero + +.Loop_is_zero_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret +.size vec_is_zero_16x,.-vec_is_zero_16x +.globl vec_is_equal_16x +.hidden vec_is_equal_16x +.type vec_is_equal_16x,%function +.align 5 +vec_is_equal_16x: + ld1 {v0.2d}, [x0], #16 + ld1 {v1.2d}, [x1], #16 + lsr x2, x2, #4 + eor v0.16b, v0.16b, v1.16b + +.Loop_is_equal: + sub x2, x2, #1 + cbz x2, .Loop_is_equal_done + ld1 {v1.2d}, [x0], #16 + ld1 {v2.2d}, [x1], #16 + eor v1.16b, v1.16b, v2.16b + orr v0.16b, v0.16b, v1.16b + b .Loop_is_equal + nop + +.Loop_is_equal_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret +.size vec_is_equal_16x,.-vec_is_equal_16x diff --git a/crypto/blst_src/build/elf/add_mod_384-x86_64.s b/crypto/blst_src/build/elf/add_mod_384-x86_64.s new file mode 100644 index 00000000000..39eee6d1752 --- /dev/null +++ b/crypto/blst_src/build/elf/add_mod_384-x86_64.s @@ -0,0 +1,1907 @@ +.text + +.globl add_mod_384 +.hidden add_mod_384 +.type add_mod_384,@function +.align 32 +add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __add_mod_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size add_mod_384,.-add_mod_384 + +.type __add_mod_384,@function +.align 32 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__add_mod_384_a_is_loaded: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __add_mod_384,.-__add_mod_384 + +.globl add_mod_384x +.hidden add_mod_384x +.type add_mod_384x,@function +.align 32 +add_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 24 + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __add_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + movq 24+0(%rsp),%r15 +.cfi_restore %r15 + movq 24+8(%rsp),%r14 +.cfi_restore %r14 + movq 24+16(%rsp),%r13 +.cfi_restore %r13 + movq 24+24(%rsp),%r12 +.cfi_restore %r12 + movq 24+32(%rsp),%rbx +.cfi_restore %rbx + movq 24+40(%rsp),%rbp +.cfi_restore %rbp + leaq 24+48(%rsp),%rsp +.cfi_adjust_cfa_offset -24-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size add_mod_384x,.-add_mod_384x + + +.globl rshift_mod_384 +.hidden rshift_mod_384 +.type rshift_mod_384,@function +.align 32 +rshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +.Loop_rshift_mod_384: + call __rshift_mod_384 + decl %edx + jnz .Loop_rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size rshift_mod_384,.-rshift_mod_384 + +.type __rshift_mod_384,@function +.align 32 +__rshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rsi + movq 0(%rcx),%r14 + andq %r8,%rsi + movq 8(%rcx),%r15 + negq %rsi + movq 16(%rcx),%rax + andq %rsi,%r14 + movq 24(%rcx),%rbx + andq %rsi,%r15 + movq 32(%rcx),%rbp + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%rbx + adcq %r12,%rbp + adcq %r13,%rsi + sbbq %r13,%r13 + + shrq $1,%r14 + movq %r15,%r8 + shrq $1,%r15 + movq %rax,%r9 + shrq $1,%rax + movq %rbx,%r10 + shrq $1,%rbx + movq %rbp,%r11 + shrq $1,%rbp + movq %rsi,%r12 + shrq $1,%rsi + shlq $63,%r8 + shlq $63,%r9 + orq %r14,%r8 + shlq $63,%r10 + orq %r15,%r9 + shlq $63,%r11 + orq %rax,%r10 + shlq $63,%r12 + orq %rbx,%r11 + shlq $63,%r13 + orq %rbp,%r12 + orq %rsi,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __rshift_mod_384,.-__rshift_mod_384 + +.globl div_by_2_mod_384 +.hidden div_by_2_mod_384 +.type div_by_2_mod_384,@function +.align 32 +div_by_2_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq %rdx,%rcx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + call __rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size div_by_2_mod_384,.-div_by_2_mod_384 + + +.globl lshift_mod_384 +.hidden lshift_mod_384 +.type lshift_mod_384,@function +.align 32 +lshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +.Loop_lshift_mod_384: + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdi,%rdi + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdi + + movq (%rsp),%rdi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + decl %edx + jnz .Loop_lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size lshift_mod_384,.-lshift_mod_384 + +.type __lshift_mod_384,@function +.align 32 +__lshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __lshift_mod_384,.-__lshift_mod_384 + + +.globl mul_by_3_mod_384 +.hidden mul_by_3_mod_384 +.type mul_by_3_mod_384,@function +.align 32 +mul_by_3_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_3_mod_384,.-mul_by_3_mod_384 + +.globl mul_by_8_mod_384 +.hidden mul_by_8_mod_384 +.type mul_by_8_mod_384,@function +.align 32 +mul_by_8_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_8_mod_384,.-mul_by_8_mod_384 + + +.globl mul_by_3_mod_384x +.hidden mul_by_3_mod_384x +.type mul_by_3_mod_384x,@function +.align 32 +mul_by_3_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq (%rsp),%rsi + leaq 48(%rdi),%rdi + + movq 48(%rsi),%r8 + movq 56(%rsi),%r9 + movq 64(%rsi),%r10 + movq 72(%rsi),%r11 + movq 80(%rsi),%r12 + movq 88(%rsi),%r13 + + call __lshift_mod_384 + + movq $48,%rdx + addq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_3_mod_384x,.-mul_by_3_mod_384x + +.globl mul_by_8_mod_384x +.hidden mul_by_8_mod_384x +.type mul_by_8_mod_384x,@function +.align 32 +mul_by_8_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq (%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,48+0(%rdi) + movq %r9,48+8(%rdi) + movq %r10,48+16(%rdi) + movq %r11,48+24(%rdi) + movq %r12,48+32(%rdi) + movq %r13,48+40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_8_mod_384x,.-mul_by_8_mod_384x + + +.globl cneg_mod_384 +.hidden cneg_mod_384 +.type cneg_mod_384,@function +.align 32 +cneg_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdx +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rdx,%r8 + movq 24(%rsi),%r11 + orq %r9,%rdx + movq 32(%rsi),%r12 + orq %r10,%rdx + movq 40(%rsi),%r13 + orq %r11,%rdx + movq $-1,%rsi + orq %r12,%rdx + orq %r13,%rdx + + movq 0(%rcx),%r14 + cmovnzq %rsi,%rdx + movq 8(%rcx),%r15 + movq 16(%rcx),%rax + andq %rdx,%r14 + movq 24(%rcx),%rbx + andq %rdx,%r15 + movq 32(%rcx),%rbp + andq %rdx,%rax + movq 40(%rcx),%rsi + andq %rdx,%rbx + movq 0(%rsp),%rcx + andq %rdx,%rbp + andq %rdx,%rsi + + subq %r8,%r14 + sbbq %r9,%r15 + sbbq %r10,%rax + sbbq %r11,%rbx + sbbq %r12,%rbp + sbbq %r13,%rsi + + orq %rcx,%rcx + + cmovzq %r8,%r14 + cmovzq %r9,%r15 + cmovzq %r10,%rax + movq %r14,0(%rdi) + cmovzq %r11,%rbx + movq %r15,8(%rdi) + cmovzq %r12,%rbp + movq %rax,16(%rdi) + cmovzq %r13,%rsi + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rsi,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size cneg_mod_384,.-cneg_mod_384 + + +.globl sub_mod_384 +.hidden sub_mod_384 +.type sub_mod_384,@function +.align 32 +sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sub_mod_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sub_mod_384,.-sub_mod_384 + +.type __sub_mod_384,@function +.align 32 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384,.-__sub_mod_384 + +.globl sub_mod_384x +.hidden sub_mod_384x +.type sub_mod_384x,@function +.align 32 +sub_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 24 + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __sub_mod_384 + + movq 24+0(%rsp),%r15 +.cfi_restore %r15 + movq 24+8(%rsp),%r14 +.cfi_restore %r14 + movq 24+16(%rsp),%r13 +.cfi_restore %r13 + movq 24+24(%rsp),%r12 +.cfi_restore %r12 + movq 24+32(%rsp),%rbx +.cfi_restore %rbx + movq 24+40(%rsp),%rbp +.cfi_restore %rbp + leaq 24+48(%rsp),%rsp +.cfi_adjust_cfa_offset -24-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sub_mod_384x,.-sub_mod_384x +.globl mul_by_1_plus_i_mod_384x +.hidden mul_by_1_plus_i_mod_384x +.type mul_by_1_plus_i_mod_384x,@function +.align 32 +mul_by_1_plus_i_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $56,%rsp +.cfi_adjust_cfa_offset 56 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rbx + adcq 72(%rsi),%r11 + movq %r12,%rcx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + movq %rdi,48(%rsp) + sbbq %rdi,%rdi + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rbx + sbbq 80(%rsi),%rcx + sbbq 88(%rsi),%rbp + sbbq %rsi,%rsi + + movq %r8,0(%rsp) + movq 0(%rdx),%r8 + movq %r9,8(%rsp) + movq 8(%rdx),%r9 + movq %r10,16(%rsp) + movq 16(%rdx),%r10 + movq %r11,24(%rsp) + movq 24(%rdx),%r11 + movq %r12,32(%rsp) + andq %rsi,%r8 + movq 32(%rdx),%r12 + movq %r13,40(%rsp) + andq %rsi,%r9 + movq 40(%rdx),%r13 + andq %rsi,%r10 + andq %rsi,%r11 + andq %rsi,%r12 + andq %rsi,%r13 + movq 48(%rsp),%rsi + + addq %r8,%r14 + movq 0(%rsp),%r8 + adcq %r9,%r15 + movq 8(%rsp),%r9 + adcq %r10,%rax + movq 16(%rsp),%r10 + adcq %r11,%rbx + movq 24(%rsp),%r11 + adcq %r12,%rcx + movq 32(%rsp),%r12 + adcq %r13,%rbp + movq 40(%rsp),%r13 + + movq %r14,0(%rsi) + movq %r8,%r14 + movq %r15,8(%rsi) + movq %rax,16(%rsi) + movq %r9,%r15 + movq %rbx,24(%rsi) + movq %rcx,32(%rsi) + movq %r10,%rax + movq %rbp,40(%rsi) + + subq 0(%rdx),%r8 + movq %r11,%rbx + sbbq 8(%rdx),%r9 + sbbq 16(%rdx),%r10 + movq %r12,%rcx + sbbq 24(%rdx),%r11 + sbbq 32(%rdx),%r12 + movq %r13,%rbp + sbbq 40(%rdx),%r13 + sbbq $0,%rdi + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,48(%rsi) + cmovcq %rbx,%r11 + movq %r9,56(%rsi) + cmovcq %rcx,%r12 + movq %r10,64(%rsi) + cmovcq %rbp,%r13 + movq %r11,72(%rsi) + movq %r12,80(%rsi) + movq %r13,88(%rsi) + + movq 56+0(%rsp),%r15 +.cfi_restore %r15 + movq 56+8(%rsp),%r14 +.cfi_restore %r14 + movq 56+16(%rsp),%r13 +.cfi_restore %r13 + movq 56+24(%rsp),%r12 +.cfi_restore %r12 + movq 56+32(%rsp),%rbx +.cfi_restore %rbx + movq 56+40(%rsp),%rbp +.cfi_restore %rbp + leaq 56+48(%rsp),%rsp +.cfi_adjust_cfa_offset -56-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x +.globl sgn0_pty_mod_384 +.hidden sgn0_pty_mod_384 +.type sgn0_pty_mod_384,@function +.align 32 +sgn0_pty_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%rcx + movq 40(%rdi),%rdx + + xorq %rax,%rax + movq %r8,%rdi + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + notq %rax + andq $1,%rdi + andq $2,%rax + orq %rdi,%rax + + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0_pty_mod_384,.-sgn0_pty_mod_384 + +.globl sgn0_pty_mod_384x +.hidden sgn0_pty_mod_384x +.type sgn0_pty_mod_384x,@function +.align 32 +sgn0_pty_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 48(%rdi),%r8 + movq 56(%rdi),%r9 + movq 64(%rdi),%r10 + movq 72(%rdi),%r11 + movq 80(%rdi),%rcx + movq 88(%rdi),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + leaq 0(%rdi),%rax + xorq %rdi,%rdi + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rdi + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rdi + + movq %r8,0(%rsp) + notq %rdi + andq $1,%rbp + andq $2,%rdi + orq %rbp,%rdi + + movq 0(%rax),%r8 + movq 8(%rax),%r9 + movq 16(%rax),%r10 + movq 24(%rax),%r11 + movq 32(%rax),%rcx + movq 40(%rax),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + xorq %rax,%rax + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + movq 0(%rsp),%rbx + + notq %rax + + testq %r8,%r8 + cmovzq %rdi,%rbp + + testq %rbx,%rbx + cmovnzq %rdi,%rax + + andq $1,%rbp + andq $2,%rax + orq %rbp,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0_pty_mod_384x,.-sgn0_pty_mod_384x +.globl vec_select_32 +.hidden vec_select_32 +.type vec_select_32,@function +.align 32 +vec_select_32: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 16(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 16(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 16(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-16(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-16(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-16(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,16-16(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_32,.-vec_select_32 +.globl vec_select_48 +.hidden vec_select_48 +.type vec_select_48,@function +.align 32 +vec_select_48: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 24(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 24(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 24(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-24(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-24(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-24(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-24(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-24(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-24(%rdi) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,32-24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_48,.-vec_select_48 +.globl vec_select_96 +.hidden vec_select_96 +.type vec_select_96,@function +.align 32 +vec_select_96: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 48(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 48(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 48(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-48(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-48(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-48(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-48(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-48(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-48(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-48(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-48(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-48(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,80-48(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_96,.-vec_select_96 +.globl vec_select_192 +.hidden vec_select_192 +.type vec_select_192,@function +.align 32 +vec_select_192: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 96(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 96(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 96(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-96(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-96(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-96(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-96(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-96(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-96(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-96(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-96(%rdi) + pand %xmm4,%xmm0 + movdqu 128+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-96(%rdi) + pand %xmm4,%xmm2 + movdqu 144+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-96(%rdi) + pand %xmm4,%xmm0 + movdqu 160+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-96(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,176-96(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_192,.-vec_select_192 +.globl vec_select_144 +.hidden vec_select_144 +.type vec_select_144,@function +.align 32 +vec_select_144: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 72(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 72(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 72(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-72(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-72(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-72(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-72(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-72(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-72(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-72(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-72(%rdi) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,128-72(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_144,.-vec_select_144 +.globl vec_select_288 +.hidden vec_select_288 +.type vec_select_288,@function +.align 32 +vec_select_288: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 144(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 144(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 144(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-144(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-144(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-144(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-144(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-144(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-144(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-144(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-144(%rdi) + pand %xmm4,%xmm0 + movdqu 128+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-144(%rdi) + pand %xmm4,%xmm2 + movdqu 144+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-144(%rdi) + pand %xmm4,%xmm0 + movdqu 160+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-144(%rdi) + pand %xmm4,%xmm2 + movdqu 176+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 176+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,176-144(%rdi) + pand %xmm4,%xmm0 + movdqu 192+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 192+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,192-144(%rdi) + pand %xmm4,%xmm2 + movdqu 208+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 208+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,208-144(%rdi) + pand %xmm4,%xmm0 + movdqu 224+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 224+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,224-144(%rdi) + pand %xmm4,%xmm2 + movdqu 240+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 240+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,240-144(%rdi) + pand %xmm4,%xmm0 + movdqu 256+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 256+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,256-144(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,272-144(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_select_288,.-vec_select_288 +.globl vec_prefetch +.hidden vec_prefetch +.type vec_prefetch,@function +.align 32 +vec_prefetch: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + leaq -1(%rdi,%rsi,1),%rsi + movq $64,%rax + xorq %r8,%r8 + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + prefetchnta (%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_prefetch,.-vec_prefetch +.globl vec_is_zero_16x +.hidden vec_is_zero_16x +.type vec_is_zero_16x,@function +.align 32 +vec_is_zero_16x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%esi + movdqu (%rdi),%xmm0 + leaq 16(%rdi),%rdi + +.Loop_is_zero: + decl %esi + jz .Loop_is_zero_done + movdqu (%rdi),%xmm1 + leaq 16(%rdi),%rdi + por %xmm1,%xmm0 + jmp .Loop_is_zero + +.Loop_is_zero_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %esi + testq %rax,%rax + cmovnzl %esi,%eax + xorl $1,%eax + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_is_zero_16x,.-vec_is_zero_16x +.globl vec_is_equal_16x +.hidden vec_is_equal_16x +.type vec_is_equal_16x,@function +.align 32 +vec_is_equal_16x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%edx + movdqu (%rdi),%xmm0 + movdqu (%rsi),%xmm1 + subq %rdi,%rsi + leaq 16(%rdi),%rdi + pxor %xmm1,%xmm0 + +.Loop_is_equal: + decl %edx + jz .Loop_is_equal_done + movdqu (%rdi),%xmm1 + movdqu (%rdi,%rsi,1),%xmm2 + leaq 16(%rdi),%rdi + pxor %xmm2,%xmm1 + por %xmm1,%xmm0 + jmp .Loop_is_equal + +.Loop_is_equal_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %edx + testq %rax,%rax + cmovnzl %edx,%eax + xorl $1,%eax + .byte 0xf3,0xc3 +.cfi_endproc +.size vec_is_equal_16x,.-vec_is_equal_16x + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/add_mod_384x384-x86_64.s b/crypto/blst_src/build/elf/add_mod_384x384-x86_64.s new file mode 100644 index 00000000000..084f3d8262d --- /dev/null +++ b/crypto/blst_src/build/elf/add_mod_384x384-x86_64.s @@ -0,0 +1,252 @@ +.text + +.type __add_mod_384x384,@function +.align 32 +__add_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + addq 0(%rdx),%r8 + movq 56(%rsi),%r15 + adcq 8(%rdx),%r9 + movq 64(%rsi),%rax + adcq 16(%rdx),%r10 + movq 72(%rsi),%rbx + adcq 24(%rdx),%r11 + movq 80(%rsi),%rbp + adcq 32(%rdx),%r12 + movq 88(%rsi),%rsi + adcq 40(%rdx),%r13 + movq %r8,0(%rdi) + adcq 48(%rdx),%r14 + movq %r9,8(%rdi) + adcq 56(%rdx),%r15 + movq %r10,16(%rdi) + adcq 64(%rdx),%rax + movq %r12,32(%rdi) + movq %r14,%r8 + adcq 72(%rdx),%rbx + movq %r11,24(%rdi) + movq %r15,%r9 + adcq 80(%rdx),%rbp + movq %r13,40(%rdi) + movq %rax,%r10 + adcq 88(%rdx),%rsi + movq %rbx,%r11 + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %rbp,%r12 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%rbx + sbbq 32(%rcx),%rbp + movq %rsi,%r13 + sbbq 40(%rcx),%rsi + sbbq $0,%rdx + + cmovcq %r8,%r14 + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %r14,48(%rdi) + cmovcq %r11,%rbx + movq %r15,56(%rdi) + cmovcq %r12,%rbp + movq %rax,64(%rdi) + cmovcq %r13,%rsi + movq %rbx,72(%rdi) + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __add_mod_384x384,.-__add_mod_384x384 + +.type __sub_mod_384x384,@function +.align 32 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384x384,.-__sub_mod_384x384 + +.globl add_mod_384x384 +.hidden add_mod_384x384 +.type add_mod_384x384,@function +.align 32 +add_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __add_mod_384x384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size add_mod_384x384,.-add_mod_384x384 + +.globl sub_mod_384x384 +.hidden sub_mod_384x384 +.type sub_mod_384x384,@function +.align 32 +sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sub_mod_384x384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sub_mod_384x384,.-sub_mod_384x384 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/ct_inverse_mod_256-armv8.S b/crypto/blst_src/build/elf/ct_inverse_mod_256-armv8.S new file mode 100644 index 00000000000..347eb315f40 --- /dev/null +++ b/crypto/blst_src/build/elf/ct_inverse_mod_256-armv8.S @@ -0,0 +1,784 @@ +.text + +.globl ct_inverse_mod_256 +.type ct_inverse_mod_256, %function +.align 5 +ct_inverse_mod_256: + .inst 0xd503233f + stp x29, x30, [sp,#-80]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + sub sp, sp, #1040 + + ldp x4, x5, [x1,#8*0] + ldp x6, x7, [x1,#8*2] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + str x0, [sp] + + ldp x8, x9, [x2,#8*0] + ldp x10, x11, [x2,#8*2] + + stp x4, x5, [x1,#8*0] // copy input to |a| + stp x6, x7, [x1,#8*2] + stp x8, x9, [x1,#8*4] // copy modulus to |b| + stp x10, x11, [x1,#8*6] + + ////////////////////////////////////////// first iteration + bl .Lab_approximation_31_256_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + str x12,[x0,#8*8] // initialize |u| with |f0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to dst |b| + bl __smul_256_n_shift_by_31 + str x12, [x0,#8*9] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + ldr x8, [x1,#8*8] // |u| + ldr x9, [x1,#8*13] // |v| + madd x4, x16, x8, xzr // |u|*|f0| + madd x4, x17, x9, x4 // |v|*|g0| + str x4, [x0,#8*4] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*5] + stp x5, x5, [x0,#8*7] + + madd x4, x12, x8, xzr // |u|*|f1| + madd x4, x13, x9, x4 // |v|*|g1| + str x4, [x0,#8*9] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*10] + stp x5, x5, [x0,#8*12] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + ////////////////////////////////////////// two[!] last iterations + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #47 // 31 + 512 % 31 + //bl __ab_approximation_62_256 // |a| and |b| are exact, + ldr x7, [x1,#8*0] // just load + ldr x11, [x1,#8*4] + bl __inner_loop_62_256 + + mov x16, x14 + mov x17, x15 + ldr x0, [sp] // original out_ptr + bl __smul_256x63 + bl __smul_512x63_tail + ldr x30, [x29,#8] + + smulh x20, x7, x17 // figure out top-most limb + ldp x8, x9, [x3,#8*0] + adc x23, x23, x25 + ldp x10, x11, [x3,#8*2] + + add x20, x20, x23 // x20 is 1, 0 or -1 + asr x19, x20, #63 // sign as mask + + and x23, x8, x19 // add mod<<256 conditionally + and x24, x9, x19 + adds x4, x4, x23 + and x25, x10, x19 + adcs x5, x5, x24 + and x26, x11, x19 + adcs x6, x6, x25 + adcs x7, x22, x26 + adc x20, x20, xzr // x20 is 1, 0 or -1 + + neg x19, x20 + orr x20, x20, x19 // excess bit or sign as mask + asr x19, x19, #63 // excess bit as mask + + and x8, x8, x20 // mask |mod| + and x9, x9, x20 + and x10, x10, x20 + and x11, x11, x20 + + eor x8, x8, x19 // conditionally negate |mod| + eor x9, x9, x19 + adds x8, x8, x19, lsr#63 + eor x10, x10, x19 + adcs x9, x9, xzr + eor x11, x11, x19 + adcs x10, x10, xzr + adc x11, x11, xzr + + adds x4, x4, x8 // final adjustment for |mod|<<256 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*4] + adc x7, x7, x11 + stp x6, x7, [x0,#8*6] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldr x29, [sp],#80 + .inst 0xd50323bf + ret +.size ct_inverse_mod_256,.-ct_inverse_mod_256 + +//////////////////////////////////////////////////////////////////////// +.type __smul_256x63, %function +.align 5 +__smul_256x63: + ldp x4, x5, [x1,#8*0+64] // load |u| (or |v|) + asr x14, x16, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x6, x7, [x1,#8*2+64] + eor x16, x16, x14 // conditionally negate |f_| (or |g_|) + ldr x22, [x1,#8*4+64] + + eor x4, x4, x14 // conditionally negate |u| (or |v|) + sub x16, x16, x14 + eor x5, x5, x14 + adds x4, x4, x14, lsr#63 + eor x6, x6, x14 + adcs x5, x5, xzr + eor x7, x7, x14 + adcs x6, x6, xzr + eor x22, x22, x14 + umulh x19, x4, x16 + adcs x7, x7, xzr + umulh x20, x5, x16 + adcs x22, x22, xzr + umulh x21, x6, x16 + mul x4, x4, x16 + cmp x16, #0 + mul x5, x5, x16 + csel x22, x22, xzr, ne + mul x6, x6, x16 + adds x5, x5, x19 + mul x24, x7, x16 + adcs x6, x6, x20 + adcs x24, x24, x21 + adc x26, xzr, xzr + ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x10, x11, [x1,#8*2+104] + eor x17, x17, x14 // conditionally negate |f_| (or |g_|) + ldr x23, [x1,#8*4+104] + + eor x8, x8, x14 // conditionally negate |u| (or |v|) + sub x17, x17, x14 + eor x9, x9, x14 + adds x8, x8, x14, lsr#63 + eor x10, x10, x14 + adcs x9, x9, xzr + eor x11, x11, x14 + adcs x10, x10, xzr + eor x23, x23, x14 + umulh x19, x8, x17 + adcs x11, x11, xzr + umulh x20, x9, x17 + adcs x23, x23, xzr + umulh x21, x10, x17 + adc x15, xzr, xzr // used in __smul_512x63_tail + mul x8, x8, x17 + cmp x17, #0 + mul x9, x9, x17 + csel x23, x23, xzr, ne + mul x10, x10, x17 + adds x9, x9, x19 + mul x25, x11, x17 + adcs x10, x10, x20 + adcs x25, x25, x21 + adc x26, x26, xzr + + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*0] + adcs x24, x24, x25 + stp x6, x24, [x0,#8*2] + + ret +.size __smul_256x63,.-__smul_256x63 + +.type __smul_512x63_tail, %function +.align 5 +__smul_512x63_tail: + umulh x24, x7, x16 + ldp x5, x6, [x1,#8*18] // load rest of |v| + adc x26, x26, xzr + ldr x7, [x1,#8*20] + and x22, x22, x16 + + umulh x11, x11, x17 // resume |v|*|g1| chain + + sub x24, x24, x22 // tie up |u|*|f1| chain + asr x25, x24, #63 + + eor x5, x5, x14 // conditionally negate rest of |v| + eor x6, x6, x14 + adds x5, x5, x15 + eor x7, x7, x14 + adcs x6, x6, xzr + umulh x19, x23, x17 + adc x7, x7, xzr + umulh x20, x5, x17 + add x11, x11, x26 + umulh x21, x6, x17 + + mul x4, x23, x17 + mul x5, x5, x17 + adds x4, x4, x11 + mul x6, x6, x17 + adcs x5, x5, x19 + mul x22, x7, x17 + adcs x6, x6, x20 + adcs x22, x22, x21 + adc x23, xzr, xzr // used in the final step + + adds x4, x4, x24 + adcs x5, x5, x25 + adcs x6, x6, x25 + stp x4, x5, [x0,#8*4] + adcs x22, x22, x25 // carry is used in the final step + stp x6, x22, [x0,#8*6] + + ret +.size __smul_512x63_tail,.-__smul_512x63_tail + +.type __smul_256_n_shift_by_31, %function +.align 5 +__smul_256_n_shift_by_31: + ldp x4, x5, [x1,#8*0+0] // load |a| (or |b|) + asr x24, x12, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x6, x7, [x1,#8*2+0] + eor x25, x12, x24 // conditionally negate |f0| (or |g0|) + + eor x4, x4, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x5, x5, x24 + adds x4, x4, x24, lsr#63 + eor x6, x6, x24 + adcs x5, x5, xzr + eor x7, x7, x24 + umulh x19, x4, x25 + adcs x6, x6, xzr + umulh x20, x5, x25 + adc x7, x7, xzr + umulh x21, x6, x25 + and x24, x24, x25 + umulh x22, x7, x25 + neg x24, x24 + + mul x4, x4, x25 + mul x5, x5, x25 + mul x6, x6, x25 + adds x5, x5, x19 + mul x7, x7, x25 + adcs x6, x6, x20 + adcs x7, x7, x21 + adc x22, x22, x24 + ldp x8, x9, [x1,#8*0+32] // load |a| (or |b|) + asr x24, x13, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x10, x11, [x1,#8*2+32] + eor x25, x13, x24 // conditionally negate |f0| (or |g0|) + + eor x8, x8, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x9, x9, x24 + adds x8, x8, x24, lsr#63 + eor x10, x10, x24 + adcs x9, x9, xzr + eor x11, x11, x24 + umulh x19, x8, x25 + adcs x10, x10, xzr + umulh x20, x9, x25 + adc x11, x11, xzr + umulh x21, x10, x25 + and x24, x24, x25 + umulh x23, x11, x25 + neg x24, x24 + + mul x8, x8, x25 + mul x9, x9, x25 + mul x10, x10, x25 + adds x9, x9, x19 + mul x11, x11, x25 + adcs x10, x10, x20 + adcs x11, x11, x21 + adc x23, x23, x24 + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + adcs x7, x7, x11 + adc x8, x22, x23 + + extr x4, x5, x4, #31 + extr x5, x6, x5, #31 + extr x6, x7, x6, #31 + asr x23, x8, #63 // result's sign as mask + extr x7, x8, x7, #31 + + eor x4, x4, x23 // ensure the result is positive + eor x5, x5, x23 + adds x4, x4, x23, lsr#63 + eor x6, x6, x23 + adcs x5, x5, xzr + eor x7, x7, x23 + adcs x6, x6, xzr + stp x4, x5, [x0,#8*0] + adc x7, x7, xzr + stp x6, x7, [x0,#8*2] + + eor x12, x12, x23 // adjust |f/g| accordingly + eor x13, x13, x23 + sub x12, x12, x23 + sub x13, x13, x23 + + ret +.size __smul_256_n_shift_by_31,.-__smul_256_n_shift_by_31 +.type __ab_approximation_31_256, %function +.align 4 +__ab_approximation_31_256: + ldp x6, x7, [x1,#8*2] + ldp x10, x11, [x1,#8*6] + ldp x4, x5, [x1,#8*0] + ldp x8, x9, [x1,#8*4] + +.Lab_approximation_31_256_loaded: + orr x19, x7, x11 // check top-most limbs, ... + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x5, ne + orr x19, x7, x11 // and ones before top-most, ... + csel x10, x10, x9, ne + + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x4, ne + orr x19, x7, x11 // and one more, ... + csel x10, x10, x8, ne + + clz x19, x19 + cmp x19, #64 + csel x19, x19, xzr, ne + csel x7, x7, x6, ne + csel x11, x11, x10, ne + neg x20, x19 + + lslv x7, x7, x19 // align high limbs to the left + lslv x11, x11, x19 + lsrv x6, x6, x20 + lsrv x10, x10, x20 + and x6, x6, x20, asr#6 + and x10, x10, x20, asr#6 + orr x7, x7, x6 + orr x11, x11, x10 + + bfxil x7, x4, #0, #31 + bfxil x11, x8, #0, #31 + + b __inner_loop_31_256 + ret +.size __ab_approximation_31_256,.-__ab_approximation_31_256 + +.type __inner_loop_31_256, %function +.align 4 +__inner_loop_31_256: + mov x2, #31 + mov x13, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x15, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x23,#0x7FFFFFFF7FFFFFFF + +.Loop_31_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x15 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x15, x15, x13, hs // exchange |fg0| and |fg1| + csel x13, x13, x19, hs + lsr x7, x7, #1 + and x19, x15, x22 + and x20, x23, x22 + sub x13, x13, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x15, x15, x15 // |f1|<<=1 + add x13, x13, x20 + sub x15, x15, x23 + cbnz x2, .Loop_31_256 + + mov x23, #0x7FFFFFFF + ubfx x12, x13, #0, #32 + ubfx x13, x13, #32, #32 + ubfx x14, x15, #0, #32 + ubfx x15, x15, #32, #32 + sub x12, x12, x23 // remove bias + sub x13, x13, x23 + sub x14, x14, x23 + sub x15, x15, x23 + + ret +.size __inner_loop_31_256,.-__inner_loop_31_256 + +.type __inner_loop_62_256, %function +.align 4 +__inner_loop_62_256: + mov x12, #1 // |f0|=1 + mov x13, #0 // |g0|=0 + mov x14, #0 // |f1|=0 + mov x15, #1 // |g1|=1 + +.Loop_62_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x12 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + mov x20, x13 + csel x12, x12, x14, hs // exchange |f0| and |f1| + csel x14, x14, x19, hs + csel x13, x13, x15, hs // exchange |g0| and |g1| + csel x15, x15, x20, hs + lsr x7, x7, #1 + and x19, x14, x22 + and x20, x15, x22 + add x14, x14, x14 // |f1|<<=1 + add x15, x15, x15 // |g1|<<=1 + sub x12, x12, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x13, x13, x20 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62_256 + + ret +.size __inner_loop_62_256,.-__inner_loop_62_256 diff --git a/crypto/blst_src/build/elf/ct_inverse_mod_256-x86_64.s b/crypto/blst_src/build/elf/ct_inverse_mod_256-x86_64.s new file mode 100644 index 00000000000..c4d8d6d3700 --- /dev/null +++ b/crypto/blst_src/build/elf/ct_inverse_mod_256-x86_64.s @@ -0,0 +1,1185 @@ +.text + +.globl ct_inverse_mod_256 +.type ct_inverse_mod_256,@function +.align 32 +ct_inverse_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1072,%rsp +.cfi_adjust_cfa_offset 1072 + + + leaq 48+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + movq 0(%rdx),%r12 + movq 8(%rdx),%r13 + movq 16(%rdx),%r14 + movq 24(%rdx),%r15 + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + + movq %r12,32(%rax) + movq %r13,40(%rax) + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rax,%rsi + + + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,64(%rdi) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,72(%rdi) + + + xorq $256,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + + movq 64(%rsi),%r8 + movq 104(%rsi),%r12 + movq %r8,%r9 + imulq 0(%rsp),%r8 + movq %r12,%r13 + imulq 8(%rsp),%r12 + addq %r12,%r8 + movq %r8,32(%rdi) + sarq $63,%r8 + movq %r8,40(%rdi) + movq %r8,48(%rdi) + movq %r8,56(%rdi) + movq %r8,64(%rdi) + leaq 64(%rsi),%rsi + + imulq %rdx,%r9 + imulq %rcx,%r13 + addq %r13,%r9 + movq %r9,72(%rdi) + sarq $63,%r9 + movq %r9,80(%rdi) + movq %r9,88(%rdi) + movq %r9,96(%rdi) + movq %r9,104(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + sarq $63,%rbp + movq %rbp,40(%rdi) + movq %rbp,48(%rdi) + movq %rbp,56(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + + xorq $256+64,%rsi + movl $47,%edx + + movq 0(%rsi),%r8 + + movq 32(%rsi),%r10 + + call __inner_loop_62_256 + + + + + + + + leaq 64(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_512x63 + adcq %rbp,%rdx + + movq 40(%rsp),%rsi + movq %rdx,%rax + sarq $63,%rdx + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + adcq $0,%rax + + movq %rax,%rdx + negq %rax + orq %rax,%rdx + sarq $63,%rax + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + xorq %rax,%r8 + xorq %rcx,%rcx + xorq %rax,%r9 + subq %rax,%rcx + xorq %rax,%r10 + xorq %rax,%rdx + addq %rcx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + leaq 1072(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1072-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size ct_inverse_mod_256,.-ct_inverse_mod_256 +.type __smulq_512x63,@function +.align 32 +__smulq_512x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %r9,8(%rdi) + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %r10,16(%rdi) + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %r11,24(%rdi) + + movq 40(%rsi),%r8 + movq 48(%rsi),%r9 + movq 56(%rsi),%r10 + movq 64(%rsi),%r11 + movq 72(%rsi),%r12 + movq 80(%rsi),%r13 + movq 88(%rsi),%r14 + movq 96(%rsi),%r15 + + movq %rcx,%rdx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rcx + addq %rax,%rcx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rcx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rcx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rcx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rcx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rcx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rcx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rcx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + imulq %rcx + addq %rax,%r15 + adcq $0,%rdx + + movq %rbp,%rbx + sarq $63,%rbp + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq %rbx,%r12 + adcq %rbp,%r13 + adcq %rbp,%r14 + adcq %rbp,%r15 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_512x63,.-__smulq_512x63 + +.type __smulq_256x63,@function +.align 32 +__smulq_256x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %rcx,%rdx + movq 40+0(%rsi),%r12 + movq 40+8(%rsi),%r13 + movq 40+16(%rsi),%r14 + movq 40+24(%rsi),%r15 + movq 40+32(%rsi),%rcx + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rcx + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rcx + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + andq %rbx,%rcx + negq %rcx + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %rbp,32(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_256x63,.-__smulq_256x63 +.type __smulq_256_n_shift_by_31,@function +.align 32 +__smulq_256_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,0(%rdi) + movq %rcx,8(%rdi) + movq %rdx,%rbp + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + + movq %rbp,%rbx + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rbx + addq %rax,%rbx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + andq %rbx,%rbp + negq %rbp + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r14 + movq 32+24(%rsi),%r15 + + movq %rcx,%rbx + sarq $63,%rcx + xorq %rax,%rax + subq %rcx,%rax + + xorq %rcx,%rbx + addq %rax,%rbx + + xorq %rcx,%r12 + xorq %rcx,%r13 + xorq %rcx,%r14 + xorq %rcx,%r15 + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + andq %rbx,%rcx + negq %rcx + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq 0(%rdi),%rdx + movq 8(%rdi),%rcx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%rbp,%r11 + + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + xorq %rbp,%rdx + xorq %rbp,%rcx + addq %rax,%rdx + addq %rax,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_256_n_shift_by_31,.-__smulq_256_n_shift_by_31 +.type __ab_approximation_31_256,@function +.align 32 +__ab_approximation_31_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 24(%rsi),%r9 + movq 56(%rsi),%r11 + movq 16(%rsi),%rbx + movq 48(%rsi),%rbp + movq 8(%rsi),%r8 + movq 40(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 32(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + notq %rax + andq %rax,%r9 + andq %rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31_256 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __ab_approximation_31_256,.-__ab_approximation_31_256 +.type __inner_loop_31_256,@function +.align 32 +__inner_loop_31_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +.Loop_31_256: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edx + jnz .Loop_31_256 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_31_256,.-__inner_loop_31_256 + +.type __inner_loop_62_256,@function +.align 32 +__inner_loop_62_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movl %edx,%r15d + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq %rdx,%r13 + movq %rdx,%r14 + +.Loop_62_256: + xorq %rax,%rax + testq %r14,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq %r14,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%r15d + jnz .Loop_62_256 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_62_256,.-__inner_loop_62_256 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/ct_inverse_mod_384-armv8.S b/crypto/blst_src/build/elf/ct_inverse_mod_384-armv8.S new file mode 100644 index 00000000000..d7eca17073c --- /dev/null +++ b/crypto/blst_src/build/elf/ct_inverse_mod_384-armv8.S @@ -0,0 +1,717 @@ +.text + +.globl ct_inverse_mod_383 +.type ct_inverse_mod_383, %function +.align 5 +ct_inverse_mod_383: + .inst 0xd503233f + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #1040 + + ldp x22, x4, [x1,#8*0] + ldp x5, x6, [x1,#8*2] + ldp x7, x8, [x1,#8*4] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + stp x0, x3, [sp] + + ldp x9, x10, [x2,#8*0] + ldp x11, x12, [x2,#8*2] + ldp x13, x14, [x2,#8*4] + + stp x22, x4, [x1,#8*0] // copy input to |a| + stp x5, x6, [x1,#8*2] + stp x7, x8, [x1,#8*4] + stp x9, x10, [x1,#8*6] // copy modulus to |b| + stp x11, x12, [x1,#8*8] + stp x13, x14, [x1,#8*10] + + ////////////////////////////////////////// first iteration + mov x2, #62 + bl .Lab_approximation_62_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + str x15,[x0,#8*12] // initialize |u| with |f0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to dst |b| + bl __smul_383_n_shift_by_62 + str x15, [x0,#8*12] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + ldr x7, [x1,#8*12] // |u| + ldr x8, [x1,#8*18] // |v| + mul x3, x20, x7 // |u|*|f0| + smulh x4, x20, x7 + mul x5, x21, x8 // |v|*|g0| + smulh x6, x21, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*6] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*8] + stp x5, x5, [x0,#8*10] + + mul x3, x15, x7 // |u|*|f1| + smulh x4, x15, x7 + mul x5, x16, x8 // |v|*|g1| + smulh x6, x16, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*12] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*14] + stp x5, x5, [x0,#8*16] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + asr x27, x27, #63 // sign extension + stp x27, x27, [x0,#8*6] + stp x27, x27, [x0,#8*8] + stp x27, x27, [x0,#8*10] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + ////////////////////////////////////////// iteration before last + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldp x3, x8, [x1,#8*0] // just load + ldp x9, x14, [x1,#8*6] + bl __inner_loop_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + str x3, [x0,#8*0] + str x9, [x0,#8*6] + + mov x20, x15 // exact |f0| + mov x21, x16 // exact |g0| + mov x15, x17 + mov x16, x19 + add x0, x0, #8*12 // pointer to dst |u| + bl __smul_383x63 + + mov x20, x15 // exact |f1| + mov x21, x16 // exact |g1| + add x0, x0, #8*6 // pointer to dst |v| + bl __smul_383x63 + bl __smul_767x63_tail + + ////////////////////////////////////////// last iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #22 // 766 % 62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldr x3, [x1,#8*0] // just load + eor x8, x8, x8 + ldr x9, [x1,#8*6] + eor x14, x14, x14 + bl __inner_loop_62 + + mov x20, x17 + mov x21, x19 + ldp x0, x15, [sp] // original out_ptr and n_ptr + bl __smul_383x63 + bl __smul_767x63_tail + ldr x30, [x29,#8] + + asr x22, x8, #63 // sign as mask + ldp x9, x10, [x15,#8*0] + ldp x11, x12, [x15,#8*2] + ldp x13, x14, [x15,#8*4] + + and x9, x9, x22 // add mod<<384 conditionally + and x10, x10, x22 + adds x3, x3, x9 + and x11, x11, x22 + adcs x4, x4, x10 + and x12, x12, x22 + adcs x5, x5, x11 + and x13, x13, x22 + adcs x6, x6, x12 + and x14, x14, x22 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*8] + adc x8, x8, x14 + stp x7, x8, [x0,#8*10] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 + .inst 0xd50323bf + ret +.size ct_inverse_mod_383,.-ct_inverse_mod_383 + +//////////////////////////////////////////////////////////////////////// +// see corresponding commentary in ctx_inverse_mod_384-x86_64... +.type __smul_383x63, %function +.align 5 +__smul_383x63: + ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) + asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x5, x6, [x1,#8*2+96] + eor x20, x20, x17 // conditionally negate |f_| (or |g_|) + ldp x7, x8, [x1,#8*4+96] + + eor x3, x3, x17 // conditionally negate |u| (or |v|) + sub x20, x20, x17 + eor x4, x4, x17 + adds x3, x3, x17, lsr#63 + eor x5, x5, x17 + adcs x4, x4, xzr + eor x6, x6, x17 + adcs x5, x5, xzr + eor x7, x7, x17 + adcs x6, x6, xzr + umulh x22, x3, x20 + eor x8, x8, x17 + umulh x23, x4, x20 + adcs x7, x7, xzr + umulh x24, x5, x20 + adcs x8, x8, xzr + umulh x25, x6, x20 + umulh x26, x7, x20 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x22 + mul x6, x6, x20 + adcs x5, x5, x23 + mul x7, x7, x20 + adcs x6, x6, x24 + mul x27,x8, x20 + adcs x7, x7, x25 + adcs x27,x27,x26 + adc x2, xzr, xzr + ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x11, x12, [x1,#8*2+144] + eor x21, x21, x17 // conditionally negate |f_| (or |g_|) + ldp x13, x14, [x1,#8*4+144] + + eor x9, x9, x17 // conditionally negate |u| (or |v|) + sub x21, x21, x17 + eor x10, x10, x17 + adds x9, x9, x17, lsr#63 + eor x11, x11, x17 + adcs x10, x10, xzr + eor x12, x12, x17 + adcs x11, x11, xzr + eor x13, x13, x17 + adcs x12, x12, xzr + umulh x22, x9, x21 + eor x14, x14, x17 + umulh x23, x10, x21 + adcs x13, x13, xzr + umulh x24, x11, x21 + adcs x14, x14, xzr + umulh x25, x12, x21 + adc x19, xzr, xzr // used in __smul_767x63_tail + umulh x26, x13, x21 + mul x9, x9, x21 + mul x10, x10, x21 + mul x11, x11, x21 + adds x10, x10, x22 + mul x12, x12, x21 + adcs x11, x11, x23 + mul x13, x13, x21 + adcs x12, x12, x24 + mul x28,x14, x21 + adcs x13, x13, x25 + adcs x28,x28,x26 + adc x2, x2, xzr + + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + stp x3, x4, [x0,#8*0] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*2] + adcs x27, x27, x28 + stp x7, x27, [x0,#8*4] + adc x28, x2, xzr // used in __smul_767x63_tail + + ret +.size __smul_383x63,.-__smul_383x63 + +.type __smul_767x63_tail, %function +.align 5 +__smul_767x63_tail: + smulh x27, x8, x20 + ldp x3, x4, [x1,#8*24] // load rest of |v| + umulh x14,x14, x21 + ldp x5, x6, [x1,#8*26] + ldp x7, x8, [x1,#8*28] + + eor x3, x3, x17 // conditionally negate rest of |v| + eor x4, x4, x17 + eor x5, x5, x17 + adds x3, x3, x19 + eor x6, x6, x17 + adcs x4, x4, xzr + eor x7, x7, x17 + adcs x5, x5, xzr + eor x8, x8, x17 + adcs x6, x6, xzr + umulh x22, x3, x21 + adcs x7, x7, xzr + umulh x23, x4, x21 + adc x8, x8, xzr + + umulh x24, x5, x21 + add x14, x14, x28 + umulh x25, x6, x21 + asr x28, x27, #63 + umulh x26, x7, x21 + mul x3, x3, x21 + mul x4, x4, x21 + mul x5, x5, x21 + adds x3, x3, x14 + mul x6, x6, x21 + adcs x4, x4, x22 + mul x7, x7, x21 + adcs x5, x5, x23 + mul x8, x8, x21 + adcs x6, x6, x24 + adcs x7, x7, x25 + adc x8, x8, x26 + + adds x3, x3, x27 + adcs x4, x4, x28 + adcs x5, x5, x28 + adcs x6, x6, x28 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x28 + stp x5, x6, [x0,#8*8] + adc x8, x8, x28 + stp x7, x8, [x0,#8*10] + + ret +.size __smul_767x63_tail,.-__smul_767x63_tail + +.type __smul_383_n_shift_by_62, %function +.align 5 +__smul_383_n_shift_by_62: + ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) + asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x5, x6, [x1,#8*2+0] + eor x2, x15, x28 // conditionally negate |f0| (or |g0|) + ldp x7, x8, [x1,#8*4+0] + + eor x3, x3, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + umulh x22, x3, x2 + adcs x6, x6, xzr + umulh x23, x4, x2 + eor x8, x8, x28 + umulh x24, x5, x2 + adcs x7, x7, xzr + umulh x25, x6, x2 + adc x8, x8, xzr + + umulh x26, x7, x2 + smulh x27, x8, x2 + mul x3, x3, x2 + mul x4, x4, x2 + mul x5, x5, x2 + adds x4, x4, x22 + mul x6, x6, x2 + adcs x5, x5, x23 + mul x7, x7, x2 + adcs x6, x6, x24 + mul x8, x8, x2 + adcs x7, x7, x25 + adcs x8, x8 ,x26 + adc x27, x27, xzr + ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) + asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x11, x12, [x1,#8*2+48] + eor x2, x16, x28 // conditionally negate |f0| (or |g0|) + ldp x13, x14, [x1,#8*4+48] + + eor x9, x9, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x10, x10, x28 + adds x9, x9, x28, lsr#63 + eor x11, x11, x28 + adcs x10, x10, xzr + eor x12, x12, x28 + adcs x11, x11, xzr + eor x13, x13, x28 + umulh x22, x9, x2 + adcs x12, x12, xzr + umulh x23, x10, x2 + eor x14, x14, x28 + umulh x24, x11, x2 + adcs x13, x13, xzr + umulh x25, x12, x2 + adc x14, x14, xzr + + umulh x26, x13, x2 + smulh x28, x14, x2 + mul x9, x9, x2 + mul x10, x10, x2 + mul x11, x11, x2 + adds x10, x10, x22 + mul x12, x12, x2 + adcs x11, x11, x23 + mul x13, x13, x2 + adcs x12, x12, x24 + mul x14, x14, x2 + adcs x13, x13, x25 + adcs x14, x14 ,x26 + adc x28, x28, xzr + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x27, x28 + + extr x3, x4, x3, #62 + extr x4, x5, x4, #62 + extr x5, x6, x5, #62 + asr x28, x9, #63 + extr x6, x7, x6, #62 + extr x7, x8, x7, #62 + extr x8, x9, x8, #62 + + eor x3, x3, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + adcs x6, x6, xzr + eor x8, x8, x28 + stp x3, x4, [x0,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x0,#8*2] + adc x8, x8, xzr + stp x7, x8, [x0,#8*4] + + eor x15, x15, x28 + eor x16, x16, x28 + sub x15, x15, x28 + sub x16, x16, x28 + + ret +.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.type __ab_approximation_62, %function +.align 4 +__ab_approximation_62: + ldp x7, x8, [x1,#8*4] + ldp x13, x14, [x1,#8*10] + ldp x5, x6, [x1,#8*2] + ldp x11, x12, [x1,#8*8] + +.Lab_approximation_62_loaded: + orr x22, x8, x14 // check top-most limbs, ... + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x22, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + ldp x3, x4, [x1,#8*0] + ldp x9, x10, [x1,#8*6] + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x22, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x22, x8, x14 + csel x13, x13, x10, ne + + clz x22, x22 + cmp x22, #64 + csel x22, x22, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x23, x22 + + lslv x8, x8, x22 // align high limbs to the left + lslv x14, x14, x22 + lsrv x7, x7, x23 + lsrv x13, x13, x23 + and x7, x7, x23, asr#6 + and x13, x13, x23, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + b __inner_loop_62 + ret +.size __ab_approximation_62,.-__ab_approximation_62 +.type __inner_loop_62, %function +.align 4 +__inner_loop_62: + mov x15, #1 // |f0|=1 + mov x16, #0 // |g0|=0 + mov x17, #0 // |f1|=0 + mov x19, #1 // |g1|=1 + +.Loop_62: + sbfx x28, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + subs x24, x9, x3 // |b_|-|a_| + and x22, x9, x28 + sbc x25, x14, x8 + and x23, x14, x28 + subs x26, x3, x22 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x22, x15 + sbcs x27, x8, x23 + mov x23, x16 + csel x9, x9, x3, hs // |b_| = |a_| + csel x14, x14, x8, hs + csel x3, x26, x24, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x8, x27, x25, hs + csel x15, x15, x17, hs // exchange |f0| and |f1| + csel x17, x17, x22, hs + csel x16, x16, x19, hs // exchange |g0| and |g1| + csel x19, x19, x23, hs + extr x3, x8, x3, #1 + lsr x8, x8, #1 + and x22, x17, x28 + and x23, x19, x28 + add x17, x17, x17 // |f1|<<=1 + add x19, x19, x19 // |g1|<<=1 + sub x15, x15, x22 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x16, x16, x23 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, .Loop_62 + + ret +.size __inner_loop_62,.-__inner_loop_62 diff --git a/crypto/blst_src/build/elf/ct_is_square_mod_384-armv8.S b/crypto/blst_src/build/elf/ct_is_square_mod_384-armv8.S new file mode 100644 index 00000000000..3f1390ed9dc --- /dev/null +++ b/crypto/blst_src/build/elf/ct_is_square_mod_384-armv8.S @@ -0,0 +1,324 @@ +.text + +.globl ct_is_square_mod_384 +.type ct_is_square_mod_384, %function +.align 5 +ct_is_square_mod_384: + .inst 0xd503233f + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #512 + + ldp x3, x4, [x0,#8*0] // load input + ldp x5, x6, [x0,#8*2] + ldp x7, x8, [x0,#8*4] + + add x0, sp, #255 // find closest 256-byte-aligned spot + and x0, x0, #-256 // in the frame... + + ldp x9, x10, [x1,#8*0] // load modulus + ldp x11, x12, [x1,#8*2] + ldp x13, x14, [x1,#8*4] + + stp x3, x4, [x0,#8*6] // copy input to |a| + stp x5, x6, [x0,#8*8] + stp x7, x8, [x0,#8*10] + stp x9, x10, [x0,#8*0] // copy modulus to |b| + stp x11, x12, [x0,#8*2] + stp x13, x14, [x0,#8*4] + + eor x2, x2, x2 // init the .Legendre symbol + mov x15, #24 // 24 is 768/30-1 + b .Loop_is_square + +.align 4 +.Loop_is_square: + bl __ab_approximation_30 + sub x15, x15, #1 + + eor x1, x0, #128 // pointer to dst |b| + bl __smul_384_n_shift_by_30 + + mov x19, x16 // |f0| + mov x20, x17 // |g0| + add x1, x1, #8*6 // pointer to dst |a| + bl __smul_384_n_shift_by_30 + + ldp x9, x10, [x1,#-8*6] + eor x0, x0, #128 // flip-flop src |a|b| + and x27, x27, x9 // if |a| was negative, + add x2, x2, x27, lsr#1 // adjust |L| + + cbnz x15, .Loop_is_square + + ////////////////////////////////////////// last iteration + //bl __ab_approximation_30 // |a| and |b| are exact, + //ldr x8, [x0,#8*6] // and loaded + //ldr x14, [x0,#8*0] + mov x15, #48 // 48 is 768%30 + 30 + bl __inner_loop_48 + ldr x30, [x29,#8] + + and x0, x2, #1 + eor x0, x0, #1 + + add sp, sp, #512 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 + .inst 0xd50323bf + ret +.size ct_is_square_mod_384,.-ct_is_square_mod_384 + +.type __smul_384_n_shift_by_30, %function +.align 5 +__smul_384_n_shift_by_30: + ldp x3, x4, [x0,#8*0+0] // load |b| (or |a|) + asr x27, x20, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x5, x6, [x0,#8*2+0] + eor x20, x20, x27 // conditionally negate |g1| (or |f1|) + ldp x7, x8, [x0,#8*4+0] + + eor x3, x3, x27 // conditionally negate |b| (or |a|) + sub x20, x20, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + umulh x21, x3, x20 + adcs x6, x6, xzr + umulh x22, x4, x20 + eor x8, x8, x27 + umulh x23, x5, x20 + adcs x7, x7, xzr + umulh x24, x6, x20 + adc x8, x8, xzr + + umulh x25, x7, x20 + and x28, x20, x27 + umulh x26, x8, x20 + neg x28, x28 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x21 + mul x6, x6, x20 + adcs x5, x5, x22 + mul x7, x7, x20 + adcs x6, x6, x23 + mul x8, x8, x20 + adcs x7, x7, x24 + adcs x8, x8 ,x25 + adc x26, x26, x28 + ldp x9, x10, [x0,#8*0+48] // load |b| (or |a|) + asr x27, x19, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x11, x12, [x0,#8*2+48] + eor x19, x19, x27 // conditionally negate |g1| (or |f1|) + ldp x13, x14, [x0,#8*4+48] + + eor x9, x9, x27 // conditionally negate |b| (or |a|) + sub x19, x19, x27 + eor x10, x10, x27 + adds x9, x9, x27, lsr#63 + eor x11, x11, x27 + adcs x10, x10, xzr + eor x12, x12, x27 + adcs x11, x11, xzr + eor x13, x13, x27 + umulh x21, x9, x19 + adcs x12, x12, xzr + umulh x22, x10, x19 + eor x14, x14, x27 + umulh x23, x11, x19 + adcs x13, x13, xzr + umulh x24, x12, x19 + adc x14, x14, xzr + + umulh x25, x13, x19 + and x28, x19, x27 + umulh x27, x14, x19 + neg x28, x28 + mul x9, x9, x19 + mul x10, x10, x19 + mul x11, x11, x19 + adds x10, x10, x21 + mul x12, x12, x19 + adcs x11, x11, x22 + mul x13, x13, x19 + adcs x12, x12, x23 + mul x14, x14, x19 + adcs x13, x13, x24 + adcs x14, x14 ,x25 + adc x27, x27, x28 + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x26, x27 + + extr x3, x4, x3, #30 + extr x4, x5, x4, #30 + extr x5, x6, x5, #30 + asr x27, x9, #63 + extr x6, x7, x6, #30 + extr x7, x8, x7, #30 + extr x8, x9, x8, #30 + + eor x3, x3, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + adcs x6, x6, xzr + eor x8, x8, x27 + stp x3, x4, [x1,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x1,#8*2] + adc x8, x8, xzr + stp x7, x8, [x1,#8*4] + + ret +.size __smul_384_n_shift_by_30,.-__smul_384_n_shift_by_30 +.type __ab_approximation_30, %function +.align 4 +__ab_approximation_30: + ldp x13, x14, [x0,#8*4] // |a| is still in registers + ldp x11, x12, [x0,#8*2] + + orr x21, x8, x14 // check top-most limbs, ... + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x21, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x21, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x21, x8, x14 // and one more, ... + csel x13, x13, x10, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x3, ne + orr x21, x8, x14 + csel x13, x13, x9, ne + + clz x21, x21 + cmp x21, #64 + csel x21, x21, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x22, x21 + + lslv x8, x8, x21 // align high limbs to the left + lslv x14, x14, x21 + lsrv x7, x7, x22 + lsrv x13, x13, x22 + and x7, x7, x22, asr#6 + and x13, x13, x22, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + bfxil x8, x3, #0, #32 + bfxil x14, x9, #0, #32 + + b __inner_loop_30 + ret +.size __ab_approximation_30,.-__ab_approximation_30 + +.type __inner_loop_30, %function +.align 4 +__inner_loop_30: + mov x28, #30 + mov x17, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x20, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x27,#0x7FFFFFFF7FFFFFFF + +.Loop_30: + sbfx x24, x8, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x8, x14 + sub x28, x28, #1 + and x21, x14, x24 + + sub x22, x14, x8 // |b_|-|a_| + subs x23, x8, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 // L + (a_ & b_) >> 1 + mov x21, x20 + csel x14, x14, x8, hs // |b_| = |a_| + csel x8, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x20, x20, x17, hs // exchange |fg0| and |fg1| + csel x17, x17, x21, hs + csel x2, x2, x25, hs + lsr x8, x8, #1 + and x21, x20, x24 + and x22, x27, x24 + add x23, x14, #2 + sub x17, x17, x21 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x20, x20, x20 // |f1|<<=1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + add x17, x17, x22 + sub x20, x20, x27 + + cbnz x28, .Loop_30 + + mov x27, #0x7FFFFFFF + ubfx x16, x17, #0, #32 + ubfx x17, x17, #32, #32 + ubfx x19, x20, #0, #32 + ubfx x20, x20, #32, #32 + sub x16, x16, x27 // remove the bias + sub x17, x17, x27 + sub x19, x19, x27 + sub x20, x20, x27 + + ret +.size __inner_loop_30,.-__inner_loop_30 +.type __inner_loop_48, %function +.align 4 +__inner_loop_48: +.Loop_48: + sbfx x24, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x3, x9 + sub x15, x15, #1 + and x21, x9, x24 + sub x22, x9, x3 // |b_|-|a_| + subs x23, x3, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 + csel x9, x9, x3, hs // |b_| = |a_| + csel x3, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x2, x2, x25, hs + add x23, x9, #2 + lsr x3, x3, #1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + + cbnz x15, .Loop_48 + + ret +.size __inner_loop_48,.-__inner_loop_48 diff --git a/crypto/blst_src/build/elf/ct_is_square_mod_384-x86_64.s b/crypto/blst_src/build/elf/ct_is_square_mod_384-x86_64.s new file mode 100644 index 00000000000..fec1493cb12 --- /dev/null +++ b/crypto/blst_src/build/elf/ct_is_square_mod_384-x86_64.s @@ -0,0 +1,479 @@ +.text + +.globl ct_is_square_mod_384 +.type ct_is_square_mod_384,@function +.align 32 +ct_is_square_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $536,%rsp +.cfi_adjust_cfa_offset 536 + + + leaq 24+255(%rsp),%rax + andq $-256,%rax + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbx + movq 24(%rsi),%rcx + movq 32(%rsi),%rdx + movq 40(%rsi),%rdi + movq %rax,%rsi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rcx,72(%rax) + movq %rdx,80(%rax) + movq %rdi,88(%rax) + + xorq %rbp,%rbp + movl $24,%ecx + jmp .Loop_is_square + +.align 32 +.Loop_is_square: + movl %ecx,16(%rsp) + + call __ab_approximation_30 + movq %rax,0(%rsp) + movq %rbx,8(%rsp) + + movq $128+48,%rdi + xorq %rsi,%rdi + call __smulq_384_n_shift_by_30 + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq -48(%rdi),%rdi + call __smulq_384_n_shift_by_30 + + movl 16(%rsp),%ecx + xorq $128,%rsi + + andq 48(%rdi),%r14 + shrq $1,%r14 + addq %r14,%rbp + + subl $1,%ecx + jnz .Loop_is_square + + + + + movq 48(%rsi),%r9 + call __inner_loop_48 + + movq $1,%rax + andq %rbp,%rax + xorq $1,%rax + + leaq 536(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -536-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size ct_is_square_mod_384,.-ct_is_square_mod_384 + +.type __smulq_384_n_shift_by_30,@function +.align 32 +__smulq_384_n_shift_by_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r14 + andq %rbx,%r14 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r14 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r14 + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r15 + andq %rbx,%r15 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r15 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r15 + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %r15,%r14 + + shrdq $30,%r9,%r8 + shrdq $30,%r10,%r9 + shrdq $30,%r11,%r10 + shrdq $30,%r12,%r11 + shrdq $30,%r13,%r12 + shrdq $30,%r14,%r13 + + sarq $63,%r14 + xorq %rbx,%rbx + subq %r14,%rbx + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_384_n_shift_by_30,.-__smulq_384_n_shift_by_30 +.type __ab_approximation_30,@function +.align 32 +__ab_approximation_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 88(%rsi),%rbx + movq 80(%rsi),%r15 + movq 72(%rsi),%r14 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r11,%r12 + movq 64(%rsi),%r11 + cmovzq %r14,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r10,%r12 + movq 56(%rsi),%r10 + cmovzq %r11,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r9,%r12 + movq 48(%rsi),%r9 + cmovzq %r10,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r8,%r12 + cmovzq %r9,%r15 + + movq %r13,%rax + orq %rbx,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r13 + cmovzq %r9,%rbx + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%r12,%r13 + shldq %cl,%r15,%rbx + + movq $0xFFFFFFFF00000000,%rax + movl %r8d,%r8d + movl %r9d,%r9d + andq %rax,%r13 + andq %rax,%rbx + orq %r13,%r8 + orq %rbx,%r9 + + jmp __inner_loop_30 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __ab_approximation_30,.-__ab_approximation_30 +.type __inner_loop_30,@function +.align 32 +__inner_loop_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rbx + movq $0x800000007FFFFFFF,%rcx + leaq -1(%rbx),%r15 + movl $30,%edi + +.Loop_30: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbx,%r12 + movq %rcx,%r13 + movq %rbp,%r14 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rcx,%rbx + cmovbq %r12,%rcx + cmovbq %rax,%rbp + + subq %r9,%r8 + subq %rcx,%rbx + addq %r15,%rbx + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbx + cmovzq %r13,%rcx + cmovzq %r14,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rcx,%rcx + leaq (%rax,%rbp,1),%rbp + subq %r15,%rcx + + subl $1,%edi + jnz .Loop_30 + + shrq $32,%r15 + movl %ebx,%eax + shrq $32,%rbx + movl %ecx,%edx + shrq $32,%rcx + subq %r15,%rax + subq %r15,%rbx + subq %r15,%rdx + subq %r15,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_30,.-__inner_loop_30 + +.type __inner_loop_48,@function +.align 32 +__inner_loop_48: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movl $48,%edi + +.Loop_48: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbp,%r12 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rax,%rbp + + subq %r9,%r8 + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rax,%rbp + + subl $1,%edi + jnz .Loop_48 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_48,.-__inner_loop_48 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s b/crypto/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..b702262f6e5 --- /dev/null +++ b/crypto/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s @@ -0,0 +1,1195 @@ +.text + +.globl ct_inverse_mod_383 +.type ct_inverse_mod_383,@function +.align 32 +ct_inverse_mod_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1112,%rsp +.cfi_adjust_cfa_offset 1112 + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + xorq $256+96,%rsi + movl $62,%edi + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 48(%rsi),%r10 + movq 56(%rsi),%r11 + call __inner_loop_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + movq %r8,0(%rdi) + movq %r10,48(%rdi) + + + + leaq 96(%rsi),%rsi + leaq 96(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + + xorq $256+96,%rsi + movl $22,%edi + + movq 0(%rsi),%r8 + xorq %r9,%r9 + movq 48(%rsi),%r10 + xorq %r11,%r11 + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1112-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.type __smulq_767x63,@function +.align 32 +__smulq_767x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + movq %r9,8(%rdi) + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + movq %r10,16(%rdi) + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %r11,24(%rdi) + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + movq %r12,32(%rdi) + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + movq %r13,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + movq %rdx,%rsi + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rsi + addq %rax,%rsi + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rbx + xorq %rdx,%rbp + xorq %rdx,%rcx + xorq %rdx,%rdi + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulq %rsi + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rsi + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rsi + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rsi + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rsi + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rsi + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %rdx,%rbx + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + mulq %rsi + addq %rax,%rbp + movq %rcx,%rax + adcq $0,%rdx + movq %rdx,%rcx + mulq %rsi + addq %rax,%rcx + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%rdi + movq 8(%rsp),%rdx + imulq %rsi,%rax + movq 16(%rsp),%rsi + addq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_767x63,.-__smulq_767x63 +.type __smulq_383x63,@function +.align 32 +__smulq_383x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_383x63,.-__smulq_383x63 +.type __smulq_383_n_shift_by_62,@function +.align 32 +__smulq_383_n_shift_by_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq 48(%rsi),%rsi + movq %rdx,%r14 + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $62,%r9,%r8 + shrdq $62,%r10,%r9 + shrdq $62,%r11,%r10 + shrdq $62,%r12,%r11 + shrdq $62,%r13,%r12 + shrdq $62,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulq_383_n_shift_by_62,.-__smulq_383_n_shift_by_62 +.type __ab_approximation_62,@function +.align 32 +__ab_approximation_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 16(%rsi),%r8 + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 8(%rsi),%r8 + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 0(%rsi),%r8 + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + jmp __inner_loop_62 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __ab_approximation_62,.-__ab_approximation_62 +.type __inner_loop_62,@function +.align 8 +.long 0 +__inner_loop_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + movq %rsi,8(%rsp) + +.Loop_62: + xorq %rax,%rax + xorq %rbx,%rbx + testq $1,%r8 + movq %r10,%rbp + movq %r11,%r14 + cmovnzq %r10,%rax + cmovnzq %r11,%rbx + subq %r8,%rbp + sbbq %r9,%r14 + movq %r8,%r15 + movq %r9,%rsi + subq %rax,%r8 + sbbq %rbx,%r9 + cmovcq %rbp,%r8 + cmovcq %r14,%r9 + cmovcq %r15,%r10 + cmovcq %rsi,%r11 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrdq $1,%r9,%r8 + shrq $1,%r9 + testq $1,%r15 + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz .Loop_62 + + movq 8(%rsp),%rsi + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_62,.-__inner_loop_62 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s b/crypto/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..25a5fa5345f --- /dev/null +++ b/crypto/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s @@ -0,0 +1,1574 @@ +.text + +.globl ctx_inverse_mod_383 +.type ctx_inverse_mod_383,@function +.align 32 +ctx_inverse_mod_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1112,%rsp +.cfi_adjust_cfa_offset 1112 + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + + xorq $256+96,%rsi + movl $53,%edi + + movq 0(%rsi),%r8 + + movq 48(%rsi),%r10 + + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulx_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1112-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size ctx_inverse_mod_383,.-ctx_inverse_mod_383 +.type __smulx_767x63,@function +.align 32 +__smulx_767x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + movq %rcx,%rax + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + sarq $63,%rax + xorq %rsi,%rsi + subq %rax,%rsi + + xorq %rax,%rdx + addq %rsi,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %rax,%r13 + xorq %rax,%r14 + xorq %rax,%r15 + xorq %rax,%rbx + xorq %rax,%rbp + xorq %rax,%rcx + xorq %rax,%rdi + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulxq %r8,%r8,%rax + mulxq %r9,%r9,%rsi + addq %rax,%r9 + mulxq %r10,%r10,%rax + adcq %rsi,%r10 + mulxq %r11,%r11,%rsi + adcq %rax,%r11 + mulxq %r12,%r12,%rax + adcq %rsi,%r12 + mulxq %r13,%r13,%rsi + adcq %rax,%r13 + mulxq %r14,%r14,%rax + adcq %rsi,%r14 + mulxq %r15,%r15,%rsi + adcq %rax,%r15 + mulxq %rbx,%rbx,%rax + adcq %rsi,%rbx + mulxq %rbp,%rbp,%rsi + adcq %rax,%rbp + mulxq %rcx,%rcx,%rax + adcq %rsi,%rcx + mulxq %rdi,%rdi,%rsi + movq 8(%rsp),%rdx + movq 16(%rsp),%rsi + adcq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulx_767x63,.-__smulx_767x63 +.type __smulx_383x63,@function +.align 32 +__smulx_383x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + movq %rcx,%rdx + adcq %rbp,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulx_383x63,.-__smulx_383x63 +.type __smulx_383_n_shift_by_31,@function +.align 32 +__smulx_383_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + xorq %r14,%r14 + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq %rdx,%r14 + + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%rax + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%r12,%r11 + shrdq $31,%rax,%r12 + shrdq $31,%r14,%rax + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulx_383_n_shift_by_31,.-__smulx_383_n_shift_by_31 +.type __smulx_191_n_shift_by_31,@function +.align 32 +__smulx_191_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %r10,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r10 + addq %rbp,%r9 + adcq $0,%r10 + imulq %rdx + addq %rax,%r10 + adcq $0,%rdx + movq %rdx,%r14 + movq %rcx,%rdx + movq 48+0(%rsi),%r11 + movq 48+8(%rsi),%r12 + movq 48+16(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r11,%r11,%rbp + mulxq %r12,%r12,%r13 + addq %rbp,%r12 + adcq $0,%r13 + imulq %rdx + addq %rax,%r13 + adcq $0,%rdx + addq %r8,%r11 + adcq %r9,%r12 + adcq %r10,%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r12,%r11 + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r11,0(%rdi) + movq %r12,8(%rdi) + movq %r13,16(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc +.size __smulx_191_n_shift_by_31,.-__smulx_191_n_shift_by_31 +.type __ab_approximation_31,@function +.align 32 +__ab_approximation_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 16(%rsi),%r8 + cmovzq %r10,%rbp + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 8(%rsi),%r8 + cmovzq %r10,%rbp + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + andnq %r9,%rax,%r9 + andnq %r11,%rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __ab_approximation_31,.-__ab_approximation_31 +.type __inner_loop_31,@function +.align 32 +__inner_loop_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +.Loop_31: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edi + jnz .Loop_31 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_31,.-__inner_loop_31 + +.type __inner_loop_62,@function +.align 32 +__inner_loop_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + +.Loop_62: + xorq %rax,%rax + testq $1,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq $1,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz .Loop_62 + + .byte 0xf3,0xc3 +.cfi_endproc +.size __inner_loop_62,.-__inner_loop_62 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/div3w-armv8.S b/crypto/blst_src/build/elf/div3w-armv8.S new file mode 100644 index 00000000000..a2b1d676a36 --- /dev/null +++ b/crypto/blst_src/build/elf/div3w-armv8.S @@ -0,0 +1,88 @@ +.text + +.globl div_3_limbs +.type div_3_limbs,%function +.align 5 +div_3_limbs: + ldp x4,x5,[x0] // load R + eor x0,x0,x0 // Q = 0 + mov x3,#64 // loop counter + nop + +.Loop: + subs x6,x4,x1 // R - D + add x0,x0,x0 // Q <<= 1 + sbcs x7,x5,x2 + add x0,x0,#1 // Q + speculative bit + csel x4,x4,x6,lo // select between R and R - D + extr x1,x2,x1,#1 // D >>= 1 + csel x5,x5,x7,lo + lsr x2,x2,#1 + sbc x0,x0,xzr // subtract speculative bit + sub x3,x3,#1 + cbnz x3,.Loop + + asr x3,x0,#63 // top bit -> mask + add x0,x0,x0 // Q <<= 1 + subs x6,x4,x1 // R - D + add x0,x0,#1 // Q + specilative bit + sbcs x7,x5,x2 + sbc x0,x0,xzr // subtract speculative bit + + orr x0,x0,x3 // all ones if overflow + + ret +.size div_3_limbs,.-div_3_limbs +.globl quot_rem_128 +.type quot_rem_128,%function +.align 5 +quot_rem_128: + ldp x3,x4,[x1] + + mul x5,x3,x2 // divisor[0:1} * quotient + umulh x6,x3,x2 + mul x11, x4,x2 + umulh x7,x4,x2 + + ldp x8,x9,[x0] // load 3 limbs of the dividend + ldr x10,[x0,#16] + + adds x6,x6,x11 + adc x7,x7,xzr + + subs x8,x8,x5 // dividend - divisor * quotient + sbcs x9,x9,x6 + sbcs x10,x10,x7 + sbc x5,xzr,xzr // borrow -> mask + + add x2,x2,x5 // if borrowed, adjust the quotient ... + and x3,x3,x5 + and x4,x4,x5 + adds x8,x8,x3 // ... and add divisor + adc x9,x9,x4 + + stp x8,x9,[x0] // save 2 limbs of the remainder + str x2,[x0,#16] // and one limb of the quotient + + mov x0,x2 // return adjusted quotient + + ret +.size quot_rem_128,.-quot_rem_128 + +.globl quot_rem_64 +.type quot_rem_64,%function +.align 5 +quot_rem_64: + ldr x3,[x1] + ldr x8,[x0] // load 1 limb of the dividend + + mul x5,x3,x2 // divisor * quotient + + sub x8,x8,x5 // dividend - divisor * quotient + + stp x8,x2,[x0] // save remainder and quotient + + mov x0,x2 // return quotient + + ret +.size quot_rem_64,.-quot_rem_64 diff --git a/crypto/blst_src/build/elf/div3w-x86_64.s b/crypto/blst_src/build/elf/div3w-x86_64.s new file mode 100644 index 00000000000..00ae5699824 --- /dev/null +++ b/crypto/blst_src/build/elf/div3w-x86_64.s @@ -0,0 +1,123 @@ +.text + +.globl div_3_limbs +.hidden div_3_limbs +.type div_3_limbs,@function +.align 32 +div_3_limbs: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq (%rdi),%r8 + movq 8(%rdi),%r9 + xorq %rax,%rax + movl $64,%ecx + +.Loop: + movq %r8,%r10 + subq %rsi,%r8 + movq %r9,%r11 + sbbq %rdx,%r9 + leaq 1(%rax,%rax,1),%rax + movq %rdx,%rdi + cmovcq %r10,%r8 + cmovcq %r11,%r9 + sbbq $0,%rax + shlq $63,%rdi + shrq $1,%rsi + shrq $1,%rdx + orq %rdi,%rsi + subl $1,%ecx + jnz .Loop + + leaq 1(%rax,%rax,1),%rcx + sarq $63,%rax + + subq %rsi,%r8 + sbbq %rdx,%r9 + sbbq $0,%rcx + + orq %rcx,%rax + + .byte 0xf3,0xc3 +.cfi_endproc +.size div_3_limbs,.-div_3_limbs +.globl quot_rem_128 +.hidden quot_rem_128 +.type quot_rem_128,@function +.align 32 +quot_rem_128: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rax + movq %rdx,%rcx + + mulq 0(%rsi) + movq %rax,%r8 + movq %rcx,%rax + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r9 + adcq $0,%rdx + + movq 0(%rdi),%r10 + movq 8(%rdi),%r11 + movq 16(%rdi),%rax + + subq %r8,%r10 + sbbq %r9,%r11 + sbbq %rdx,%rax + sbbq %r8,%r8 + + addq %r8,%rcx + movq %r8,%r9 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + addq %r8,%r10 + adcq %r9,%r11 + + movq %r10,0(%rdi) + movq %r11,8(%rdi) + movq %rcx,16(%rdi) + + movq %rcx,%rax + + .byte 0xf3,0xc3 +.cfi_endproc +.size quot_rem_128,.-quot_rem_128 + + + + + +.globl quot_rem_64 +.hidden quot_rem_64 +.type quot_rem_64,@function +.align 32 +quot_rem_64: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rax + imulq 0(%rsi),%rdx + + movq 0(%rdi),%r10 + + subq %rdx,%r10 + + movq %r10,0(%rdi) + movq %rax,8(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size quot_rem_64,.-quot_rem_64 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/mul_mont_256-armv8.S b/crypto/blst_src/build/elf/mul_mont_256-armv8.S new file mode 100644 index 00000000000..8bb1197f464 --- /dev/null +++ b/crypto/blst_src/build/elf/mul_mont_256-armv8.S @@ -0,0 +1,464 @@ +.text + +.globl mul_mont_sparse_256 +.hidden mul_mont_sparse_256 +.type mul_mont_sparse_256,%function +.align 5 +mul_mont_sparse_256: + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x10,x11,[x1] + ldr x9, [x2] + ldp x12,x13,[x1,#16] + + mul x19,x10,x9 + ldp x5,x6,[x3] + mul x20,x11,x9 + ldp x7,x8,[x3,#16] + mul x21,x12,x9 + mul x22,x13,x9 + + umulh x14,x10,x9 + umulh x15,x11,x9 + mul x3,x4,x19 + umulh x16,x12,x9 + umulh x17,x13,x9 + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,xzr, x17 + mul x17,x8,x3 + ldr x9,[x2,8*1] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*2] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*3] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + adcs x20,x21,x15 + adcs x21,x22,x16 + adcs x22,x23,x17 + adc x23,xzr,xzr + + subs x14,x19,x5 + sbcs x15,x20,x6 + sbcs x16,x21,x7 + sbcs x17,x22,x8 + sbcs xzr, x23,xzr + + csel x19,x19,x14,lo + csel x20,x20,x15,lo + csel x21,x21,x16,lo + csel x22,x22,x17,lo + + stp x19,x20,[x0] + stp x21,x22,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + ret +.size mul_mont_sparse_256,.-mul_mont_sparse_256 +.globl sqr_mont_sparse_256 +.hidden sqr_mont_sparse_256 +.type sqr_mont_sparse_256,%function +.align 5 +sqr_mont_sparse_256: + .inst 0xd503233f + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + mov x4,x3 + + //////////////////////////////////////////////////////////////// + // | | | | | |a1*a0| | + // | | | | |a2*a0| | | + // | |a3*a2|a3*a0| | | | + // | | | |a2*a1| | | | + // | | |a3*a1| | | | | + // *| | | | | | | | 2| + // +|a3*a3|a2*a2|a1*a1|a0*a0| + // |--+--+--+--+--+--+--+--| + // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is x10 + // + // "can't overflow" below mark carrying into high part of + // multiplication result, which can't overflow, because it + // can never be all ones. + + mul x11,x6,x5 // a[1]*a[0] + umulh x15,x6,x5 + mul x12,x7,x5 // a[2]*a[0] + umulh x16,x7,x5 + mul x13,x8,x5 // a[3]*a[0] + umulh x19,x8,x5 + + adds x12,x12,x15 // accumulate high parts of multiplication + mul x14,x7,x6 // a[2]*a[1] + umulh x15,x7,x6 + adcs x13,x13,x16 + mul x16,x8,x6 // a[3]*a[1] + umulh x17,x8,x6 + adc x19,x19,xzr // can't overflow + + mul x20,x8,x7 // a[3]*a[2] + umulh x21,x8,x7 + + adds x15,x15,x16 // accumulate high parts of multiplication + mul x10,x5,x5 // a[0]*a[0] + adc x16,x17,xzr // can't overflow + + adds x13,x13,x14 // accumulate low parts of multiplication + umulh x5,x5,x5 + adcs x19,x19,x15 + mul x15,x6,x6 // a[1]*a[1] + adcs x20,x20,x16 + umulh x6,x6,x6 + adc x21,x21,xzr // can't overflow + + adds x11,x11,x11 // acc[1-6]*=2 + mul x16,x7,x7 // a[2]*a[2] + adcs x12,x12,x12 + umulh x7,x7,x7 + adcs x13,x13,x13 + mul x17,x8,x8 // a[3]*a[3] + adcs x19,x19,x19 + umulh x8,x8,x8 + adcs x20,x20,x20 + adcs x21,x21,x21 + adc x22,xzr,xzr + + adds x11,x11,x5 // +a[i]*a[i] + adcs x12,x12,x15 + adcs x13,x13,x6 + adcs x19,x19,x16 + adcs x20,x20,x7 + adcs x21,x21,x17 + adc x22,x22,x8 + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + adds x10,x10,x19 // accumulate upper half + adcs x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adc x19,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x19,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + .inst 0xd50323bf + ret +.size sqr_mont_sparse_256,.-sqr_mont_sparse_256 +.globl from_mont_256 +.hidden from_mont_256 +.type from_mont_256,%function +.align 5 +from_mont_256: + .inst 0xd503233f + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 + .inst 0xd50323bf + ret +.size from_mont_256,.-from_mont_256 + +.globl redc_mont_256 +.hidden redc_mont_256 +.type redc_mont_256,%function +.align 5 +redc_mont_256: + .inst 0xd503233f + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + ldp x14,x15,[x1,#32] + ldp x16,x17,[x1,#48] + + adds x10,x10,x14 + adcs x11,x11,x15 + adcs x12,x12,x16 + adcs x13,x13,x17 + adc x9,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x9,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 + .inst 0xd50323bf + ret +.size redc_mont_256,.-redc_mont_256 + +.type __mul_by_1_mont_256,%function +.align 5 +__mul_by_1_mont_256: + mul x3,x4,x10 + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + adc x13,x9,x17 + + ret +.size __mul_by_1_mont_256,.-__mul_by_1_mont_256 diff --git a/crypto/blst_src/build/elf/mul_mont_384-armv8.S b/crypto/blst_src/build/elf/mul_mont_384-armv8.S new file mode 100644 index 00000000000..c048e816b85 --- /dev/null +++ b/crypto/blst_src/build/elf/mul_mont_384-armv8.S @@ -0,0 +1,2372 @@ +.text + +.globl add_mod_384x384 +.type add_mod_384x384,%function +.align 5 +add_mod_384x384: + .inst 0xd503233f + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __add_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + .inst 0xd50323bf + ret +.size add_mod_384x384,.-add_mod_384x384 + +.type __add_mod_384x384,%function +.align 5 +__add_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + stp x11, x12, [x0] + adcs x15,x15,x23 + ldp x11, x12, [x1,#48] + adcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + adcs x11,x11,x19 + stp x15, x16, [x0,#32] + adcs x12,x12,x20 + ldp x15, x16, [x1,#80] + adcs x13,x13,x21 + ldp x23,x24,[x2,#80] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + stp x11,x12,[x0,#48] + csel x15,x15,x23,lo + stp x13,x14,[x0,#64] + csel x16,x16,x24,lo + stp x15,x16,[x0,#80] + + ret +.size __add_mod_384x384,.-__add_mod_384x384 + +.globl sub_mod_384x384 +.type sub_mod_384x384,%function +.align 5 +sub_mod_384x384: + .inst 0xd503233f + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + .inst 0xd50323bf + ret +.size sub_mod_384x384,.-sub_mod_384x384 + +.type __sub_mod_384x384,%function +.align 5 +__sub_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + stp x11, x12, [x0] + sbcs x15,x15,x23 + ldp x11, x12, [x1,#48] + sbcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + sbcs x11,x11,x19 + stp x15, x16, [x0,#32] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#80] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#80] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + ret +.size __sub_mod_384x384,.-__sub_mod_384x384 + +.type __add_mod_384,%function +.align 5 +__add_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + stp x11,x12,[x0] + csel x16,x16,x24,lo + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret +.size __add_mod_384,.-__add_mod_384 + +.type __sub_mod_384,%function +.align 5 +__sub_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0] + adc x16,x16,x24 + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret +.size __sub_mod_384,.-__sub_mod_384 + +.globl mul_mont_384x +.hidden mul_mont_384x +.type mul_mont_384x,%function +.align 5 +mul_mont_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#288 // space for 3 768-bit vectors + + mov x26,x0 // save r_ptr + mov x27,x1 // save b_ptr + mov x28,x2 // save b_ptr + + sub x0,sp,#0 // mul_384(t0, a->re, b->re) + bl __mul_384 + + add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x2,x2,#48 + add x0,sp,#96 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + sub x2,x1,#48 + add x0,sp,#240 + bl __add_mod_384 + + add x1,x28,#0 + add x2,x28,#48 + add x0,sp,#192 // t2 + bl __add_mod_384 + + add x1,x0,#0 + add x2,x0,#48 + bl __mul_384 // mul_384(t2, a->re+a->im, b->re+b->im) + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,x0 + add x2,sp,#0 + bl __sub_mod_384x384 + + add x2,sp,#96 + bl __sub_mod_384x384 // t2 = t2-t0-t1 + + add x1,sp,#0 + add x2,sp,#96 + add x0,sp,#0 + bl __sub_mod_384x384 // t0 = t0-t1 + + add x1,sp,#0 // ret->re = redc(t0) + add x0,x26,#0 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + + add x1,sp,#192 // ret->im = redc(t2) + add x0,x0,#48 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#288 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size mul_mont_384x,.-mul_mont_384x + +.globl sqr_mont_384x +.hidden sqr_mont_384x +.type sqr_mont_384x,%function +.align 5 +sqr_mont_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 2 384-bit vectors + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + add x0,sp,#0 + bl __add_mod_384 // t0 = a->re + a->im + + add x0,sp,#48 + bl __sub_mod_384 // t1 = a->re - a->im + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __mul_mont_384 // mul_mont_384(ret->im, a->re, a->im) + + adds x11,x11,x11 // add with itself + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x19,x11,x19,lo + csel x20,x12,x20,lo + csel x21,x13,x21,lo + ldp x11,x12,[sp] + csel x22,x14,x22,lo + ldr x17, [sp,#48] + csel x23,x15,x23,lo + ldp x13,x14,[sp,#16] + csel x24,x16,x24,lo + ldp x15,x16,[sp,#32] + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + add x2,sp,#48 + bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_mont_384x,.-sqr_mont_384x + +.globl mul_mont_384 +.hidden mul_mont_384 +.type mul_mont_384,%function +.align 5 +mul_mont_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size mul_mont_384,.-mul_mont_384 + +.type __mul_mont_384,%function +.align 5 +__mul_mont_384: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + mov x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*1] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*2] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*3] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*4] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*5] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + adc x17,x17,xzr + + adds x19,x20,x26 + adcs x20,x21,x27 + adcs x21,x22,x28 + adcs x22,x23,x0 + adcs x23,x24,x1 + adcs x24,x25,x3 + adc x25,x17,xzr + + subs x26,x19,x5 + sbcs x27,x20,x6 + sbcs x28,x21,x7 + sbcs x0,x22,x8 + sbcs x1,x23,x9 + sbcs x3,x24,x10 + sbcs xzr, x25,xzr + + csel x11,x19,x26,lo + csel x12,x20,x27,lo + csel x13,x21,x28,lo + csel x14,x22,x0,lo + csel x15,x23,x1,lo + csel x16,x24,x3,lo + ret +.size __mul_mont_384,.-__mul_mont_384 + +.globl sqr_mont_384 +.hidden sqr_mont_384 +.type sqr_mont_384,%function +.align 5 +sqr_mont_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for 768-bit vector + mov x4,x3 // adjust for missing b_ptr + + mov x3,x0 // save r_ptr + mov x0,sp + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + mov x1,sp + mov x0,x3 // restore r_ptr + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_mont_384,.-sqr_mont_384 + +.globl sqr_n_mul_mont_383 +.hidden sqr_n_mul_mont_383 +.type sqr_n_mul_mont_383,%function +.align 5 +sqr_n_mul_mont_383: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 768-bit vector + mov x17,x5 // save b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + mov x0,sp +.Loop_sqr_383: + bl __sqr_384 + sub x2,x2,#1 // counter + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,sp + bl __mul_by_1_mont_384 + + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // just accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + cbnz x2,.Loop_sqr_383 + + mov x2,x17 + ldr x17,[x17] + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 +.type __sqr_384,%function +.align 5 +__sqr_384: + mul x19,x12,x11 + mul x20,x13,x11 + mul x21,x14,x11 + mul x22,x15,x11 + mul x23,x16,x11 + + umulh x6,x12,x11 + umulh x7,x13,x11 + umulh x8,x14,x11 + umulh x9,x15,x11 + adds x20,x20,x6 + umulh x10,x16,x11 + adcs x21,x21,x7 + mul x7,x13,x12 + adcs x22,x22,x8 + mul x8,x14,x12 + adcs x23,x23,x9 + mul x9,x15,x12 + adc x24,xzr, x10 + mul x10,x16,x12 + + adds x21,x21,x7 + umulh x7,x13,x12 + adcs x22,x22,x8 + umulh x8,x14,x12 + adcs x23,x23,x9 + umulh x9,x15,x12 + adcs x24,x24,x10 + umulh x10,x16,x12 + adc x25,xzr,xzr + + mul x5,x11,x11 + adds x22,x22,x7 + umulh x11, x11,x11 + adcs x23,x23,x8 + mul x8,x14,x13 + adcs x24,x24,x9 + mul x9,x15,x13 + adc x25,x25,x10 + mul x10,x16,x13 + + adds x23,x23,x8 + umulh x8,x14,x13 + adcs x24,x24,x9 + umulh x9,x15,x13 + adcs x25,x25,x10 + umulh x10,x16,x13 + adc x26,xzr,xzr + + mul x6,x12,x12 + adds x24,x24,x8 + umulh x12, x12,x12 + adcs x25,x25,x9 + mul x9,x15,x14 + adc x26,x26,x10 + mul x10,x16,x14 + + adds x25,x25,x9 + umulh x9,x15,x14 + adcs x26,x26,x10 + umulh x10,x16,x14 + adc x27,xzr,xzr + mul x7,x13,x13 + adds x26,x26,x9 + umulh x13, x13,x13 + adc x27,x27,x10 + mul x8,x14,x14 + + mul x10,x16,x15 + umulh x14, x14,x14 + adds x27,x27,x10 + umulh x10,x16,x15 + mul x9,x15,x15 + adc x28,x10,xzr + + adds x19,x19,x19 + adcs x20,x20,x20 + adcs x21,x21,x21 + adcs x22,x22,x22 + adcs x23,x23,x23 + adcs x24,x24,x24 + adcs x25,x25,x25 + adcs x26,x26,x26 + umulh x15, x15,x15 + adcs x27,x27,x27 + mul x10,x16,x16 + adcs x28,x28,x28 + umulh x16, x16,x16 + adc x1,xzr,xzr + + adds x19,x19,x11 + adcs x20,x20,x6 + adcs x21,x21,x12 + adcs x22,x22,x7 + adcs x23,x23,x13 + adcs x24,x24,x8 + adcs x25,x25,x14 + stp x5,x19,[x0] + adcs x26,x26,x9 + stp x20,x21,[x0,#16] + adcs x27,x27,x15 + stp x22,x23,[x0,#32] + adcs x28,x28,x10 + stp x24,x25,[x0,#48] + adc x16,x16,x1 + stp x26,x27,[x0,#64] + stp x28,x16,[x0,#80] + + ret +.size __sqr_384,.-__sqr_384 +.globl sqr_384 +.hidden sqr_384 +.type sqr_384,%function +.align 5 +sqr_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_384,.-sqr_384 + +.globl redc_mont_384 +.hidden redc_mont_384 +.type redc_mont_384,%function +.align 5 +redc_mont_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size redc_mont_384,.-redc_mont_384 + +.globl from_mont_384 +.hidden from_mont_384 +.type from_mont_384,%function +.align 5 +from_mont_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size from_mont_384,.-from_mont_384 + +.type __mul_by_1_mont_384,%function +.align 5 +__mul_by_1_mont_384: + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + mul x26,x4,x11 + ldp x15,x16,[x1,#32] + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + ret +.size __mul_by_1_mont_384,.-__mul_by_1_mont_384 + +.type __redc_tail_mont_384,%function +.align 5 +__redc_tail_mont_384: + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret +.size __redc_tail_mont_384,.-__redc_tail_mont_384 + +.globl mul_384 +.hidden mul_384 +.type mul_384,%function +.align 5 +mul_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + bl __mul_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size mul_384,.-mul_384 + +.type __mul_384,%function +.align 5 +__mul_384: + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + + umulh x5,x11,x17 + umulh x6,x12,x17 + umulh x7,x13,x17 + umulh x8,x14,x17 + umulh x9,x15,x17 + umulh x10,x16,x17 + ldr x17,[x2,8*1] + + str x19,[x0] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,xzr, x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(1+1)] + adc x25,xzr,xzr + + str x19,[x0,8*1] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(2+1)] + adc x25,xzr,xzr + + str x19,[x0,8*2] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(3+1)] + adc x25,xzr,xzr + + str x19,[x0,8*3] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(4+1)] + adc x25,xzr,xzr + + str x19,[x0,8*4] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + adc x25,xzr,xzr + + str x19,[x0,8*5] + adds x19,x20,x5 + adcs x20,x21,x6 + adcs x21,x22,x7 + adcs x22,x23,x8 + adcs x23,x24,x9 + adc x24,x25,x10 + + stp x19,x20,[x0,#48] + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ret +.size __mul_384,.-__mul_384 + +.globl mul_382x +.hidden mul_382x +.type mul_382x,%function +.align 5 +mul_382x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for two 384-bit vectors + + ldp x11,x12,[x1] + mov x26,x0 // save r_ptr + ldp x19,x20,[x1,#48] + mov x27,x1 // save a_ptr + ldp x13,x14,[x1,#16] + mov x28,x2 // save b_ptr + ldp x21,x22,[x1,#64] + ldp x15,x16,[x1,#32] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x23,x24,[x1,#80] + adcs x6,x12,x20 + ldp x11,x12,[x2] + adcs x7,x13,x21 + ldp x19,x20,[x2,#48] + adcs x8,x14,x22 + ldp x13,x14,[x2,#16] + adcs x9,x15,x23 + ldp x21,x22,[x2,#64] + adc x10,x16,x24 + ldp x15,x16,[x2,#32] + + stp x5,x6,[sp] + adds x5,x11,x19 // t1 = b->re + b->im + ldp x23,x24,[x2,#80] + adcs x6,x12,x20 + stp x7,x8,[sp,#16] + adcs x7,x13,x21 + adcs x8,x14,x22 + stp x9,x10,[sp,#32] + adcs x9,x15,x23 + stp x5,x6,[sp,#48] + adc x10,x16,x24 + stp x7,x8,[sp,#64] + stp x9,x10,[sp,#80] + + bl __mul_384 // mul_384(ret->re, a->re, b->re) + + add x1,sp,#0 // mul_384(ret->im, t0, t1) + add x2,sp,#48 + add x0,x26,#96 + bl __mul_384 + + add x1,x27,#48 // mul_384(tx, a->im, b->im) + add x2,x28,#48 + add x0,sp,#0 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + add x1,x26,#96 // ret->im -= tx + add x2,sp,#0 + add x0,x26,#96 + bl __sub_mod_384x384 + + add x2,x26,#0 // ret->im -= ret->re + bl __sub_mod_384x384 + + add x1,x26,#0 // ret->re -= tx + add x2,sp,#0 + add x0,x26,#0 + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size mul_382x,.-mul_382x + +.globl sqr_382x +.hidden sqr_382x +.type sqr_382x,%function +.align 5 +sqr_382x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x19,x20,[x1,#48] + ldp x13,x14,[x1,#16] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x21,x22,[x1,#64] + adcs x6,x12,x20 + ldp x15,x16,[x1,#32] + adcs x7,x13,x21 + ldp x23,x24,[x1,#80] + adcs x8,x14,x22 + stp x5,x6,[x0] + adcs x9,x15,x23 + ldp x5,x6,[x2] + adc x10,x16,x24 + stp x7,x8,[x0,#16] + + subs x11,x11,x19 // t1 = a->re - a->im + ldp x7,x8,[x2,#16] + sbcs x12,x12,x20 + stp x9,x10,[x0,#32] + sbcs x13,x13,x21 + ldp x9,x10,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + adds x11,x11,x19 + and x21,x7,x25 + adcs x12,x12,x20 + and x22,x8,x25 + adcs x13,x13,x21 + and x23,x9,x25 + adcs x14,x14,x22 + and x24,x10,x25 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + mov x4,x1 // save a_ptr + add x1,x0,#0 // mul_384(ret->re, t0, t1) + add x2,x0,#48 + bl __mul_384 + + add x1,x4,#0 // mul_384(ret->im, a->re, a->im) + add x2,x4,#48 + add x0,x0,#96 + bl __mul_384 + ldr x30,[x29,#8] + + ldp x11,x12,[x0] + ldp x13,x14,[x0,#16] + adds x11,x11,x11 // add with itself + ldp x15,x16,[x0,#32] + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adcs x19,x19,x19 + adcs x20,x20,x20 + stp x11,x12,[x0] + adcs x21,x21,x21 + stp x13,x14,[x0,#16] + adcs x22,x22,x22 + stp x15,x16,[x0,#32] + adcs x23,x23,x23 + stp x19,x20,[x0,#48] + adc x24,x24,x24 + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_382x,.-sqr_382x + +.globl sqr_mont_382x +.hidden sqr_mont_382x +.type sqr_mont_382x,%function +.align 5 +sqr_mont_382x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#112 // space for two 384-bit vectors + word + mov x4,x3 // adjust for missing b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x17,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x5,x11,x17 // t0 = a->re + a->im + adcs x6,x12,x20 + adcs x7,x13,x21 + adcs x8,x14,x22 + adcs x9,x15,x23 + adc x10,x16,x24 + + subs x19,x11,x17 // t1 = a->re - a->im + sbcs x20,x12,x20 + sbcs x21,x13,x21 + sbcs x22,x14,x22 + sbcs x23,x15,x23 + sbcs x24,x16,x24 + sbc x25,xzr,xzr // borrow flag as mask + + stp x5,x6,[sp] + stp x7,x8,[sp,#16] + stp x9,x10,[sp,#32] + stp x19,x20,[sp,#48] + stp x21,x22,[sp,#64] + stp x23,x24,[sp,#80] + str x25,[sp,#96] + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) + + adds x19,x11,x11 // add with itself + adcs x20,x12,x12 + adcs x21,x13,x13 + adcs x22,x14,x14 + adcs x23,x15,x15 + adc x24,x16,x16 + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + ldp x11,x12,[sp] + ldr x17,[sp,#48] + ldp x13,x14,[sp,#16] + ldp x15,x16,[sp,#32] + + add x2,sp,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) + ldr x30,[x29,#8] + + ldr x25,[sp,#96] // account for sign from a->re - a->im + ldp x19,x20,[sp] + ldp x21,x22,[sp,#16] + ldp x23,x24,[sp,#32] + + and x19,x19,x25 + and x20,x20,x25 + and x21,x21,x25 + and x22,x22,x25 + and x23,x23,x25 + and x24,x24,x25 + + subs x11,x11,x19 + sbcs x12,x12,x20 + sbcs x13,x13,x21 + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + and x21,x7,x25 + and x22,x8,x25 + and x23,x9,x25 + and x24,x10,x25 + + adds x11,x11,x19 + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#112 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sqr_mont_382x,.-sqr_mont_382x + +.type __mul_mont_383_nonred,%function +.align 5 +__mul_mont_383_nonred: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + ldr x17,[x2,8*1] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*2] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*3] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*4] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*5] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + + adds x11,x20,x26 + adcs x12,x21,x27 + adcs x13,x22,x28 + adcs x14,x23,x0 + adcs x15,x24,x1 + adcs x16,x25,x3 + + ret +.size __mul_mont_383_nonred,.-__mul_mont_383_nonred + +.globl sgn0_pty_mont_384 +.hidden sgn0_pty_mont_384 +.type sgn0_pty_mont_384,%function +.align 5 +sgn0_pty_mont_384: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + adds x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sgn0_pty_mont_384,.-sgn0_pty_mont_384 + +.globl sgn0_pty_mont_384x +.hidden sgn0_pty_mont_384x +.type sgn0_pty_mont_384x,%function +.align 5 +sgn0_pty_mont_384x: + .inst 0xd503233f + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + add x1,x1,#48 + + and x2,x11,#1 + orr x3,x11,x12 + adds x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + orr x3,x3,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x2,x2,x17 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + orr x1,x11,x12 + adds x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + orr x1,x1,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + .inst 0xd50323bf + ret +.size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x diff --git a/crypto/blst_src/build/elf/mulq_mont_256-x86_64.s b/crypto/blst_src/build/elf/mulq_mont_256-x86_64.s new file mode 100644 index 00000000000..37abd4392d3 --- /dev/null +++ b/crypto/blst_src/build/elf/mulq_mont_256-x86_64.s @@ -0,0 +1,714 @@ +.text + +.globl mul_mont_sparse_256 +.hidden mul_mont_sparse_256 +.type mul_mont_sparse_256,@function +.align 32 +mul_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rdx),%rax + movq 0(%rsi),%r13 + movq 8(%rsi),%r14 + movq 16(%rsi),%r12 + movq 24(%rsi),%rbp + movq %rdx,%rbx + + movq %rax,%r15 + mulq %r13 + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_mont_sparse_256,.-mul_mont_sparse_256 + +.globl sqr_mont_sparse_256 +.hidden sqr_mont_sparse_256 +.type sqr_mont_sparse_256,@function +.align 32 +sqr_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rax + movq %rcx,%r8 + movq 8(%rsi),%r14 + movq %rdx,%rcx + movq 16(%rsi),%r12 + leaq (%rsi),%rbx + movq 24(%rsi),%rbp + + movq %rax,%r15 + mulq %rax + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_mont_sparse_256,.-sqr_mont_sparse_256 +.type __mulq_mont_sparse_256,@function +.align 32 +__mulq_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + mulq %r14 + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r12 + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq 8(%rbx),%rax + adcq $0,%rdx + xorq %r14,%r14 + movq %rdx,%r13 + + movq %r9,%rdi + imulq %r8,%r9 + + + movq %rax,%r15 + mulq 0(%rsi) + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + xorq %r15,%r15 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r9,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rdi,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + addq %rdx,%r13 + adcq $0,%r14 + adcq $0,%r15 + movq %r10,%rdi + imulq %r8,%r10 + + + movq %rax,%r9 + mulq 0(%rsi) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + xorq %r9,%r9 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r10,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rdi,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r13 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + addq %rdx,%r14 + adcq $0,%r15 + adcq $0,%r9 + movq %r11,%rdi + imulq %r8,%r11 + + + movq %rax,%r10 + mulq 0(%rsi) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r9 + xorq %r10,%r10 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r11,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rdi,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + addq %rdx,%r15 + adcq $0,%r9 + adcq $0,%r10 + imulq %r8,%rax + movq 8(%rsp),%rsi + + + movq %rax,%r11 + mulq 0(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r12,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + movq %r14,%rbx + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rdx,%r9 + adcq $0,%r10 + + + + + movq %r15,%r12 + subq 0(%rcx),%r13 + sbbq 8(%rcx),%r14 + sbbq 16(%rcx),%r15 + movq %r9,%rbp + sbbq 24(%rcx),%r9 + sbbq $0,%r10 + + cmovcq %rax,%r13 + cmovcq %rbx,%r14 + cmovcq %r12,%r15 + movq %r13,0(%rsi) + cmovcq %rbp,%r9 + movq %r14,8(%rsi) + movq %r15,16(%rsi) + movq %r9,24(%rsi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_mont_sparse_256,.-__mulq_mont_sparse_256 +.globl from_mont_256 +.hidden from_mont_256 +.type from_mont_256,@function +.align 32 +from_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + + + + + movq %r14,%r10 + movq %r15,%r11 + movq %r9,%r12 + + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + sbbq 24(%rbx),%r9 + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size from_mont_256,.-from_mont_256 + +.globl redc_mont_256 +.hidden redc_mont_256 +.type redc_mont_256,@function +.align 32 +redc_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + addq 32(%rsi),%r13 + adcq 40(%rsi),%r14 + movq %r13,%rax + adcq 48(%rsi),%r15 + movq %r14,%r10 + adcq 56(%rsi),%r9 + sbbq %rsi,%rsi + + + + + movq %r15,%r11 + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + movq %r9,%r12 + sbbq 24(%rbx),%r9 + sbbq $0,%rsi + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size redc_mont_256,.-redc_mont_256 +.type __mulq_by_1_mont_256,@function +.align 32 +__mulq_by_1_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + + movq %rax,%r13 + imulq %rcx,%rax + movq %rax,%r9 + + mulq 0(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq %rdx,%r13 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r10 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 16(%rbx) + movq %r10,%r14 + imulq %rcx,%r10 + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r11 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r13,%r12 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r9 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_by_1_mont_256,.-__mulq_by_1_mont_256 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/mulq_mont_384-x86_64.s b/crypto/blst_src/build/elf/mulq_mont_384-x86_64.s new file mode 100644 index 00000000000..fa9dd3529ad --- /dev/null +++ b/crypto/blst_src/build/elf/mulq_mont_384-x86_64.s @@ -0,0 +1,3620 @@ +.text + + + + + + + +.type __sub_mod_384x384,@function +.align 32 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384x384,.-__sub_mod_384x384 + +.type __add_mod_384,@function +.align 32 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __add_mod_384,.-__add_mod_384 + +.type __sub_mod_384,@function +.align 32 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384,.-__sub_mod_384 +.globl mul_mont_384x +.hidden mul_mont_384x +.type mul_mont_384x,@function +.align 32 +mul_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $328,%rsp +.cfi_adjust_cfa_offset 328 + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulq_384 + + + leaq 48(%rbx),%rbx + leaq 48(%rsi),%rsi + leaq 40+96(%rsp),%rdi + call __mulq_384 + + + movq 8(%rsp),%rcx + leaq -48(%rsi),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulq_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + movq %rcx,%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -328-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_mont_384x,.-mul_mont_384x +.globl sqr_mont_384x +.hidden sqr_mont_384x +.type sqr_mont_384x,@function +.align 32 +sqr_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + call __mulq_mont_384 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + movq %r14,%r12 + adcq %r9,%r9 + movq %r15,%r13 + adcq %r10,%r10 + movq %r8,%rax + adcq %r11,%r11 + movq %r9,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r10,%rbp + sbbq 16(%rcx),%r8 + sbbq 24(%rcx),%r9 + sbbq 32(%rcx),%r10 + movq %r11,%rsi + sbbq 40(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %r12,%r14 + cmovcq %r13,%r15 + cmovcq %rax,%r8 + movq %r14,48(%rdi) + cmovcq %rbx,%r9 + movq %r15,56(%rdi) + cmovcq %rbp,%r10 + movq %r8,64(%rdi) + cmovcq %rsi,%r11 + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_mont_384x,.-sqr_mont_384x + +.globl mul_382x +.hidden mul_382x +.type mul_382x,@function +.align 32 +mul_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulq_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulq_384 + + + leaq 48(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulq_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_382x,.-mul_382x +.globl sqr_382x +.hidden sqr_382x +.type sqr_382x,@function +.align 32 +sqr_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulq_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulq_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -8*7 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_382x,.-sqr_382x +.globl mul_384 +.hidden mul_384 +.type mul_384,@function +.align 32 +mul_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq %rdx,%rbx + call __mulq_384 + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_384,.-mul_384 + +.type __mulq_384,@function +.align 32 +__mulq_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rax + + movq %rax,%rbp + mulq 0(%rsi) + movq %rax,0(%rdi) + movq %rbp,%rax + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r11 + movq 8(%rbx),%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,8(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,16(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,24(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,32(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,40(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq %rax,%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rcx,48(%rdi) + movq %r8,56(%rdi) + movq %r9,64(%rdi) + movq %r10,72(%rdi) + movq %r11,80(%rdi) + movq %r12,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_384,.-__mulq_384 +.globl sqr_384 +.hidden sqr_384 +.type sqr_384,@function +.align 32 +sqr_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sqrq_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_384,.-sqr_384 + +.type __sqrq_384,@function +.align 32 +__sqrq_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r15 + movq 16(%rsi),%rcx + movq 24(%rsi),%rbx + + + movq %rax,%r14 + mulq %r15 + movq %rax,%r9 + movq %r14,%rax + movq 32(%rsi),%rbp + movq %rdx,%r10 + + mulq %rcx + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + movq 40(%rsi),%rsi + movq %rdx,%r11 + + mulq %rbx + addq %rax,%r11 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq %rax + xorq %r8,%r8 + movq %rax,0(%rdi) + movq %r15,%rax + addq %r9,%r9 + adcq $0,%r8 + addq %rdx,%r9 + adcq $0,%r8 + movq %r9,8(%rdi) + + mulq %rcx + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbx + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbp + addq %rax,%r13 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq %rax + xorq %r9,%r9 + addq %rax,%r8 + movq %rcx,%rax + addq %r10,%r10 + adcq %r11,%r11 + adcq $0,%r9 + addq %r8,%r10 + adcq %rdx,%r11 + adcq $0,%r9 + movq %r10,16(%rdi) + + mulq %rbx + addq %rax,%r13 + movq %rcx,%rax + adcq $0,%rdx + movq %r11,24(%rdi) + movq %rdx,%r8 + + mulq %rbp + addq %rax,%r14 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq %rsi + addq %rax,%r15 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + xorq %r11,%r11 + addq %rax,%r9 + movq %rbx,%rax + addq %r12,%r12 + adcq %r13,%r13 + adcq $0,%r11 + addq %r9,%r12 + adcq %rdx,%r13 + adcq $0,%r11 + movq %r12,32(%rdi) + + + mulq %rbp + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %r13,40(%rdi) + movq %rdx,%r8 + + mulq %rsi + addq %rax,%rcx + movq %rbx,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%rbx + + mulq %rax + xorq %r12,%r12 + addq %rax,%r11 + movq %rbp,%rax + addq %r14,%r14 + adcq %r15,%r15 + adcq $0,%r12 + addq %r11,%r14 + adcq %rdx,%r15 + movq %r14,48(%rdi) + adcq $0,%r12 + movq %r15,56(%rdi) + + + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq %rax + xorq %r13,%r13 + addq %rax,%r12 + movq %rsi,%rax + addq %rcx,%rcx + adcq %rbx,%rbx + adcq $0,%r13 + addq %r12,%rcx + adcq %rdx,%rbx + movq %rcx,64(%rdi) + adcq $0,%r13 + movq %rbx,72(%rdi) + + + mulq %rax + addq %r13,%rax + addq %rbp,%rbp + adcq $0,%rdx + addq %rbp,%rax + adcq $0,%rdx + movq %rax,80(%rdi) + movq %rdx,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sqrq_384,.-__sqrq_384 + +.globl sqr_mont_384 +.hidden sqr_mont_384 +.type sqr_mont_384,@function +.align 32 +sqr_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $120,%rsp +.cfi_adjust_cfa_offset 8*15 + + + movq %rcx,96(%rsp) + movq %rdx,104(%rsp) + movq %rdi,112(%rsp) + + movq %rsp,%rdi + call __sqrq_384 + + leaq 0(%rsp),%rsi + movq 96(%rsp),%rcx + movq 104(%rsp),%rbx + movq 112(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 120(%rsp),%r8 + movq 120(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*21 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_mont_384,.-sqr_mont_384 + + + +.globl redc_mont_384 +.hidden redc_mont_384 +.type redc_mont_384,@function +.align 32 +redc_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size redc_mont_384,.-redc_mont_384 + + + + +.globl from_mont_384 +.hidden from_mont_384 +.type from_mont_384,@function +.align 32 +from_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + + + + + + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size from_mont_384,.-from_mont_384 +.type __mulq_by_1_mont_384,@function +.align 32 +__mulq_by_1_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r8 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r8,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r9 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r10 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %r9,%r15 + imulq %rcx,%r9 + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 32(%rbx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 40(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r9,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %r10,%r8 + imulq %rcx,%r10 + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r8 + movq %r10,%rax + adcq %rdx,%r8 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %r11,%r9 + imulq %rcx,%r11 + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r11,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %r12,%r10 + imulq %rcx,%r12 + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %r13,%r11 + imulq %rcx,%r13 + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_by_1_mont_384,.-__mulq_by_1_mont_384 + +.type __redc_tail_mont_384,@function +.align 32 +__redc_tail_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __redc_tail_mont_384,.-__redc_tail_mont_384 + +.globl sgn0_pty_mont_384 +.hidden sgn0_pty_mont_384 +.type sgn0_pty_mont_384,@function +.align 32 +sgn0_pty_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0_pty_mont_384,.-sgn0_pty_mont_384 + +.globl sgn0_pty_mont_384x +.hidden sgn0_pty_mont_384x +.type sgn0_pty_mont_384x,@function +.align 32 +sgn0_pty_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x +.globl mul_mont_384 +.hidden mul_mont_384 +.type mul_mont_384,@function +.align 32 +mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq 0(%rdx),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + movq %rdx,%rbx + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + + call __mulq_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -72 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mul_mont_384,.-mul_mont_384 +.type __mulq_mont_384,@function +.align 32 +__mulq_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rdi + mulq %r14 + movq %rax,%r8 + movq %rdi,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%rbp + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + xorq %r15,%r15 + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r8,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + adcq $0,%r15 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r9,%rbp + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r14 + adcq $0,%rdx + xorq %r8,%r8 + addq %rax,%r14 + movq %r9,%rax + adcq %rdx,%r15 + adcq $0,%r8 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r9,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + adcq $0,%r8 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r10,%rbp + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r15 + adcq $0,%rdx + xorq %r9,%r9 + addq %rax,%r15 + movq %r10,%rax + adcq %rdx,%r8 + adcq $0,%r9 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r10,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r8 + adcq $0,%r9 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r11,%rbp + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r8 + adcq $0,%rdx + xorq %r10,%r10 + addq %rax,%r8 + movq %r11,%rax + adcq %rdx,%r9 + adcq $0,%r10 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r11,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq %rdx,%r9 + adcq $0,%r10 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r12,%rbp + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r9 + adcq $0,%rdx + xorq %r11,%r11 + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r10 + adcq $0,%r11 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r12,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq %rdx,%r10 + adcq $0,%r11 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + movq %r13,%rbp + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r8 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rsi) + addq %r12,%r10 + adcq $0,%rdx + xorq %r12,%r12 + addq %rax,%r10 + movq %r13,%rax + adcq %rdx,%r11 + adcq $0,%r12 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r13,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq %rdx,%r11 + adcq $0,%r12 + + + + + movq 16(%rsp),%rdi + subq 0(%rcx),%r14 + movq %r15,%rdx + sbbq 8(%rcx),%r15 + movq %r8,%rbx + sbbq 16(%rcx),%r8 + movq %r9,%rsi + sbbq 24(%rcx),%r9 + movq %r10,%rbp + sbbq 32(%rcx),%r10 + movq %r11,%r13 + sbbq 40(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rdx,%r15 + cmovcq %rbx,%r8 + movq %r14,0(%rdi) + cmovcq %rsi,%r9 + movq %r15,8(%rdi) + cmovcq %rbp,%r10 + movq %r8,16(%rdi) + cmovcq %r13,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_mont_384,.-__mulq_mont_384 +.globl sqr_n_mul_mont_384 +.hidden sqr_n_mul_mont_384 +.type sqr_n_mul_mont_384,@function +.align 32 +sqr_n_mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 8*17 + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +.Loop_sqr_384: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movd %xmm1,%edx + leaq 0(%rdi),%rsi + decl %edx + jnz .Loop_sqr_384 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*23 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_n_mul_mont_384,.-sqr_n_mul_mont_384 + +.globl sqr_n_mul_mont_383 +.hidden sqr_n_mul_mont_383 +.type sqr_n_mul_mont_383,@function +.align 32 +sqr_n_mul_mont_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 8*17 + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +.Loop_sqr_383: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + + movd %xmm1,%edx + addq 48(%rsi),%r14 + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + leaq 0(%rdi),%rsi + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + decl %edx + jnz .Loop_sqr_383 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*23 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 +.type __mulq_mont_383_nonred,@function +.align 32 +__mulq_mont_383_nonred: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rbp + mulq %r14 + movq %rax,%r8 + movq %rbp,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%r15 + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%r15 + movq %r8,%rax + adcq %rdx,%r15 + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r9 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rcx) + addq %r15,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %r15,%r13 + adcq %rdx,%r14 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + movq %r9,%r8 + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rsi) + addq %r15,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rcx) + addq %rax,%r8 + movq %r9,%rax + adcq %rdx,%r8 + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rcx) + addq %r8,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r8,%r14 + adcq %rdx,%r15 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r10,%r9 + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rcx) + addq %rax,%r9 + movq %r10,%rax + adcq %rdx,%r9 + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rcx) + addq %r9,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r9,%r15 + adcq %rdx,%r8 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r11,%r10 + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rcx) + addq %rax,%r10 + movq %r11,%rax + adcq %rdx,%r10 + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rcx) + addq %r10,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r10,%r8 + adcq %rdx,%r9 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r12,%r11 + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r9 + adcq $0,%rdx + addq %rax,%r9 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rcx) + addq %rax,%r11 + movq %r12,%rax + adcq %rdx,%r11 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rcx) + addq %r11,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r11,%r9 + adcq %rdx,%r10 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r13,%r12 + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r10 + adcq $0,%rdx + addq %rax,%r10 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 0(%rcx) + addq %rax,%r12 + movq %r13,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 24(%rcx) + addq %r12,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r12,%r10 + adcq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulq_mont_383_nonred,.-__mulq_mont_383_nonred +.globl sqr_mont_382x +.hidden sqr_mont_382x +.type sqr_mont_382x,@function +.align 32 +sqr_mont_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rsi,16(%rsp) + movq %rdi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq 24(%rsp),%rdi + call __mulq_mont_383_nonred + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %r8,64(%rdi) + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_383_nonred + movq 32+96(%rsp),%rsi + movq 32+0(%rsp),%r12 + movq 32+8(%rsp),%r13 + andq %rsi,%r12 + movq 32+16(%rsp),%rax + andq %rsi,%r13 + movq 32+24(%rsp),%rbx + andq %rsi,%rax + movq 32+32(%rsp),%rbp + andq %rsi,%rbx + andq %rsi,%rbp + andq 32+40(%rsp),%rsi + + subq %r12,%r14 + movq 0(%rcx),%r12 + sbbq %r13,%r15 + movq 8(%rcx),%r13 + sbbq %rax,%r8 + movq 16(%rcx),%rax + sbbq %rbx,%r9 + movq 24(%rcx),%rbx + sbbq %rbp,%r10 + movq 32(%rcx),%rbp + sbbq %rsi,%r11 + sbbq %rsi,%rsi + + andq %rsi,%r12 + andq %rsi,%r13 + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r12,%r14 + adcq %r13,%r15 + adcq %rax,%r8 + adcq %rbx,%r9 + adcq %rbp,%r10 + adcq %rsi,%r11 + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqr_mont_382x,.-sqr_mont_382x + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/mulx_mont_256-x86_64.s b/crypto/blst_src/build/elf/mulx_mont_256-x86_64.s new file mode 100644 index 00000000000..20a02073246 --- /dev/null +++ b/crypto/blst_src/build/elf/mulx_mont_256-x86_64.s @@ -0,0 +1,627 @@ +.text + +.globl mulx_mont_sparse_256 +.hidden mulx_mont_sparse_256 +.type mulx_mont_sparse_256,@function +.align 32 +mulx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mulx_mont_sparse_256,.-mulx_mont_sparse_256 + +.globl sqrx_mont_sparse_256 +.hidden sqrx_mont_sparse_256 +.type sqrx_mont_sparse_256,@function +.align 32 +sqrx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + movq %rcx,%r8 + movq %rdx,%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_mont_sparse_256,.-sqrx_mont_sparse_256 +.type __mulx_mont_sparse_256,@function +.align 32 +__mulx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + mulxq %r15,%r15,%r12 + mulxq %rbp,%rbp,%r13 + addq %r15,%r11 + mulxq %r9,%r9,%r14 + movq 8(%rbx),%rdx + adcq %rbp,%r12 + adcq %r9,%r13 + adcq $0,%r14 + + movq %rax,%r10 + imulq %r8,%rax + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r11 + adcxq %r9,%r12 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r14 + adcxq %r15,%r9 + adoxq %r9,%r15 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r10 + adoxq %r11,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r12 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r12 + adoxq %r9,%r13 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 16(%rbx),%rdx + adcxq %rbp,%r13 + adoxq %r9,%r14 + adcxq %r10,%r14 + adoxq %r10,%r15 + adcxq %r10,%r15 + adoxq %r10,%r10 + adcq $0,%r10 + movq %rax,%r11 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r15 + adcxq %r10,%r9 + adoxq %r9,%r10 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r11 + adoxq %r12,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r13 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r13 + adoxq %r9,%r14 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 24(%rbx),%rdx + adcxq %rbp,%r14 + adoxq %r9,%r15 + adcxq %r11,%r15 + adoxq %r11,%r10 + adcxq %r11,%r10 + adoxq %r11,%r11 + adcq $0,%r11 + movq %rax,%r12 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r15 + adcxq %r9,%r10 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r10 + adcxq %r11,%r9 + adoxq %r9,%r11 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r12 + adoxq %r13,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r14 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %rax,%rdx + adcxq %rbp,%r15 + adoxq %r9,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + adoxq %r12,%r12 + adcq $0,%r12 + imulq %r8,%rdx + + + xorq %rbp,%rbp + mulxq 0+128(%rcx),%r13,%r9 + adcxq %rax,%r13 + adoxq %r9,%r14 + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r15 + adoxq %r9,%r10 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %r14,%rdx + leaq 128(%rcx),%rcx + adcxq %rbp,%r10 + adoxq %r9,%r11 + movq %r15,%rax + adcxq %r13,%r11 + adoxq %r13,%r12 + adcq $0,%r12 + + + + + movq %r10,%rbp + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + sbbq 16(%rcx),%r10 + movq %r11,%r9 + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rdx,%r14 + cmovcq %rax,%r15 + cmovcq %rbp,%r10 + movq %r14,0(%rdi) + cmovcq %r9,%r11 + movq %r15,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_mont_sparse_256,.-__mulx_mont_sparse_256 +.globl fromx_mont_256 +.hidden fromx_mont_256 +.type fromx_mont_256,@function +.align 32 +fromx_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + + + + + movq %r15,%rdx + movq %r10,%r12 + movq %r11,%r13 + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + sbbq 24(%rbx),%r11 + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size fromx_mont_256,.-fromx_mont_256 + +.globl redcx_mont_256 +.hidden redcx_mont_256 +.type redcx_mont_256,@function +.align 32 +redcx_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + addq 32(%rsi),%r14 + adcq 40(%rsi),%r15 + movq %r14,%rax + adcq 48(%rsi),%r10 + movq %r15,%rdx + adcq 56(%rsi),%r11 + sbbq %rsi,%rsi + + + + + movq %r10,%r12 + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + movq %r11,%r13 + sbbq 24(%rbx),%r11 + sbbq $0,%rsi + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size redcx_mont_256,.-redcx_mont_256 +.type __mulx_by_1_mont_256,@function +.align 32 +__mulx_by_1_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r11 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r10 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r10 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + movq %r13,%r11 + imulq %rcx,%r13 + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_by_1_mont_256,.-__mulx_by_1_mont_256 + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/mulx_mont_384-x86_64.s b/crypto/blst_src/build/elf/mulx_mont_384-x86_64.s new file mode 100644 index 00000000000..9f9f7404ee4 --- /dev/null +++ b/crypto/blst_src/build/elf/mulx_mont_384-x86_64.s @@ -0,0 +1,2968 @@ +.text + + + + + + + +.type __sub_mod_384x384,@function +.align 32 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384x384,.-__sub_mod_384x384 + +.type __add_mod_384,@function +.align 32 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __add_mod_384,.-__add_mod_384 + +.type __sub_mod_384,@function +.align 32 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sub_mod_384,.-__sub_mod_384 +.globl mulx_mont_384x +.hidden mulx_mont_384x +.type mulx_mont_384x,@function +.align 32 +mulx_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $328,%rsp +.cfi_adjust_cfa_offset 328 + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulx_384 + + + leaq 48(%rbx),%rbx + leaq 128+48(%rsi),%rsi + leaq 96(%rdi),%rdi + call __mulx_384 + + + movq 8(%rsp),%rcx + leaq (%rbx),%rsi + leaq -48(%rbx),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulx_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + leaq (%rcx),%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -328-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mulx_mont_384x,.-mulx_mont_384x +.globl sqrx_mont_384x +.hidden sqrx_mont_384x +.type sqrx_mont_384x,@function +.align 32 +sqrx_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + movq %rdx,%r8 + adcq %r12,%r12 + movq %r15,%r9 + adcq %rdi,%rdi + movq %rax,%r10 + adcq %rbp,%rbp + movq %r12,%r11 + sbbq %rsi,%rsi + + subq 0(%rcx),%rdx + sbbq 8(%rcx),%r15 + movq %rdi,%r13 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r12 + sbbq 32(%rcx),%rdi + movq %rbp,%r14 + sbbq 40(%rcx),%rbp + sbbq $0,%rsi + + cmovcq %r8,%rdx + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %rdx,48(%rbx) + cmovcq %r11,%r12 + movq %r15,56(%rbx) + cmovcq %r13,%rdi + movq %rax,64(%rbx) + cmovcq %r14,%rbp + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_mont_384x,.-sqrx_mont_384x + +.globl mulx_382x +.hidden mulx_382x +.type mulx_382x,@function +.align 32 +mulx_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulx_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulx_384 + + + leaq 48+128(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulx_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mulx_382x,.-mulx_382x +.globl sqrx_382x +.hidden sqrx_382x +.type sqrx_382x,@function +.align 32 +sqrx_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulx_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulx_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -8*7 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_382x,.-sqrx_382x +.globl mulx_384 +.hidden mulx_384 +.type mulx_384,@function +.align 32 +mulx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + + + movq %rdx,%rbx + call __mulx_384 + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mulx_384,.-mulx_384 + +.type __mulx_384,@function +.align 32 +__mulx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + leaq -128(%rsi),%rsi + + mulxq %r14,%r9,%rcx + xorq %rbp,%rbp + + mulxq %r15,%r8,%rax + adcxq %rcx,%r8 + movq %r9,0(%rdi) + + mulxq %r10,%r9,%rcx + adcxq %rax,%r9 + + mulxq %r11,%r10,%rax + adcxq %rcx,%r10 + + mulxq %r12,%r11,%rcx + adcxq %rax,%r11 + + mulxq %r13,%r12,%r13 + movq 8(%rbx),%rdx + adcxq %rcx,%r12 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,8(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 16(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,16(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 24(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,24(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 32(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,32(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 40(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,40(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq %rax,%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + movq %r10,64(%rdi) + movq %r11,72(%rdi) + movq %r12,80(%rdi) + movq %r13,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_384,.-__mulx_384 +.globl sqrx_384 +.hidden sqrx_384 +.type sqrx_384,@function +.align 32 +sqrx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + call __sqrx_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_384,.-sqrx_384 +.type __sqrx_384,@function +.align 32 +__sqrx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rdx + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%rcx + movq 32(%rsi),%rbx + + + mulxq %r14,%r8,%rdi + movq 40(%rsi),%rbp + mulxq %r15,%r9,%rax + addq %rdi,%r9 + mulxq %rcx,%r10,%rdi + adcq %rax,%r10 + mulxq %rbx,%r11,%rax + adcq %rdi,%r11 + mulxq %rbp,%r12,%r13 + movq %r14,%rdx + adcq %rax,%r12 + adcq $0,%r13 + + + xorq %r14,%r14 + mulxq %r15,%rdi,%rax + adcxq %rdi,%r10 + adoxq %rax,%r11 + + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r11 + adoxq %rax,%r12 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbp,%rdi,%rax + movq %r15,%rdx + adcxq %rdi,%r13 + adoxq %r14,%rax + adcxq %rax,%r14 + + + xorq %r15,%r15 + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r13 + adoxq %rax,%r14 + + mulxq %rbp,%rdi,%rax + movq %rcx,%rdx + adcxq %rdi,%r14 + adoxq %r15,%rax + adcxq %rax,%r15 + + + xorq %rcx,%rcx + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r14 + adoxq %rax,%r15 + + mulxq %rbp,%rdi,%rax + movq %rbx,%rdx + adcxq %rdi,%r15 + adoxq %rcx,%rax + adcxq %rax,%rcx + + + mulxq %rbp,%rdi,%rbx + movq 0(%rsi),%rdx + addq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rbx + + + xorq %rbp,%rbp + adcxq %r8,%r8 + adcxq %r9,%r9 + adcxq %r10,%r10 + adcxq %r11,%r11 + adcxq %r12,%r12 + + + mulxq %rdx,%rdx,%rax + movq %rdx,0(%rdi) + movq 8(%rsi),%rdx + adoxq %rax,%r8 + movq %r8,8(%rdi) + + mulxq %rdx,%r8,%rax + movq 16(%rsi),%rdx + adoxq %r8,%r9 + adoxq %rax,%r10 + movq %r9,16(%rdi) + movq %r10,24(%rdi) + + mulxq %rdx,%r8,%r9 + movq 24(%rsi),%rdx + adoxq %r8,%r11 + adoxq %r9,%r12 + adcxq %r13,%r13 + adcxq %r14,%r14 + movq %r11,32(%rdi) + movq %r12,40(%rdi) + + mulxq %rdx,%r8,%r9 + movq 32(%rsi),%rdx + adoxq %r8,%r13 + adoxq %r9,%r14 + adcxq %r15,%r15 + adcxq %rcx,%rcx + movq %r13,48(%rdi) + movq %r14,56(%rdi) + + mulxq %rdx,%r8,%r9 + movq 40(%rsi),%rdx + adoxq %r8,%r15 + adoxq %r9,%rcx + adcxq %rbx,%rbx + adcxq %rbp,%rbp + movq %r15,64(%rdi) + movq %rcx,72(%rdi) + + mulxq %rdx,%r8,%r9 + adoxq %r8,%rbx + adoxq %r9,%rbp + + movq %rbx,80(%rdi) + movq %rbp,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __sqrx_384,.-__sqrx_384 + + + +.globl redcx_mont_384 +.hidden redcx_mont_384 +.type redcx_mont_384,@function +.align 32 +redcx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size redcx_mont_384,.-redcx_mont_384 + + + + +.globl fromx_mont_384 +.hidden fromx_mont_384 +.type fromx_mont_384,@function +.align 32 +fromx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + + + + + movq %r14,%rax + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size fromx_mont_384,.-fromx_mont_384 +.type __mulx_by_1_mont_384,@function +.align 32 +__mulx_by_1_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq %rcx,%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + imulq %r8,%rdx + + + xorq %r14,%r14 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r13 + adoxq %r14,%rbp + adcxq %rbp,%r14 + imulq %r9,%rdx + + + xorq %r15,%r15 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r14 + adoxq %r15,%rbp + adcxq %rbp,%r15 + imulq %r10,%rdx + + + xorq %r8,%r8 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r8 + imulq %r11,%rdx + + + xorq %r9,%r9 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r8 + adoxq %r9,%rbp + adcxq %rbp,%r9 + imulq %r12,%rdx + + + xorq %r10,%r10 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r10 + imulq %r13,%rdx + + + xorq %r11,%r11 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r11 + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_by_1_mont_384,.-__mulx_by_1_mont_384 + +.type __redc_tail_mont_384,@function +.align 32 +__redc_tail_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __redc_tail_mont_384,.-__redc_tail_mont_384 + +.globl sgn0x_pty_mont_384 +.hidden sgn0x_pty_mont_384 +.type sgn0x_pty_mont_384,@function +.align 32 +sgn0x_pty_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0x_pty_mont_384,.-sgn0x_pty_mont_384 + +.globl sgn0x_pty_mont_384x +.hidden sgn0x_pty_mont_384x +.type sgn0x_pty_mont_384x,@function +.align 32 +sgn0x_pty_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sgn0x_pty_mont_384x,.-sgn0x_pty_mont_384x +.globl mulx_mont_384 +.hidden mulx_mont_384 +.type mulx_mont_384,@function +.align 32 +mulx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -24(%rsp),%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + movq %r8,(%rsp) + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -8*9 + + .byte 0xf3,0xc3 +.cfi_endproc +.size mulx_mont_384,.-mulx_mont_384 +.type __mulx_mont_384,@function +.align 32 +__mulx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + xorq %r15,%r15 + + movq %r8,16(%rsp) + imulq 8(%rsp),%r8 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %rbp,%r15 + adoxq %rax,%r15 + adoxq %rax,%rax + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %r8,%r14 + adoxq %r8,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r9,16(%rsp) + imulq 8(%rsp),%r9 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rbp,%rax + adoxq %r8,%rax + adoxq %r8,%r8 + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r9,%r15 + adoxq %r9,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r10,16(%rsp) + imulq 8(%rsp),%r10 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %rbp,%r8 + adoxq %r9,%r8 + adoxq %r9,%r9 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r10,%rax + adoxq %r10,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r11,16(%rsp) + imulq 8(%rsp),%r11 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %rbp,%r9 + adoxq %r10,%r9 + adoxq %r10,%r10 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r11,%r8 + adoxq %r11,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + movq %r12,16(%rsp) + imulq 8(%rsp),%r12 + + + xorq %r11,%r11 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %rbp,%r10 + adoxq %r11,%r10 + adoxq %r11,%r11 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r12,%r9 + adoxq %r12,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + movq %r15,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + movq %rax,%rsi + + mulxq 40+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + movq %r14,%rdx + adcxq %r12,%r10 + adoxq %r12,%r11 + leaq 128(%rcx),%rcx + movq %r8,%r12 + adcq $0,%r11 + + + + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r9,%rdi + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r8 + sbbq 32(%rcx),%r9 + movq %r10,%rbp + sbbq 40(%rcx),%r10 + sbbq $0,%r11 + + cmovncq %r14,%rdx + cmovcq %r13,%r15 + cmovcq %rsi,%rax + cmovncq %r8,%r12 + movq %rdx,0(%rbx) + cmovncq %r9,%rdi + movq %r15,8(%rbx) + cmovncq %r10,%rbp + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_mont_384,.-__mulx_mont_384 +.globl sqrx_mont_384 +.hidden sqrx_mont_384 +.type sqrx_mont_384,@function +.align 32 +sqrx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -24(%rsp),%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq %rcx,%r8 + leaq -128(%rdx),%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + leaq (%rsi),%rbx + movq %r8,(%rsp) + leaq -128(%rsi),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -8*9 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_mont_384,.-sqrx_mont_384 + +.globl sqrx_n_mul_mont_384 +.hidden sqrx_n_mul_mont_384 +.type sqrx_n_mul_mont_384,@function +.align 32 +sqrx_n_mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -40(%rsp),%rsp +.cfi_adjust_cfa_offset 8*5 + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + +.Loop_sqrx_384: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movd %xmm1,%r10d + decl %r10d + jnz .Loop_sqrx_384 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 +.cfi_restore %r15 + movq 48(%rsp),%r14 +.cfi_restore %r14 + movq 56(%rsp),%r13 +.cfi_restore %r13 + movq 64(%rsp),%r12 +.cfi_restore %r12 + movq 72(%rsp),%rbx +.cfi_restore %rbx + movq 80(%rsp),%rbp +.cfi_restore %rbp + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -8*11 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_n_mul_mont_384,.-sqrx_n_mul_mont_384 + +.globl sqrx_n_mul_mont_383 +.hidden sqrx_n_mul_mont_383 +.type sqrx_n_mul_mont_383,@function +.align 32 +sqrx_n_mul_mont_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -40(%rsp),%rsp +.cfi_adjust_cfa_offset 8*5 + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + leaq -128(%rcx),%rcx + +.Loop_sqrx_383: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_383_nonred + + movd %xmm1,%r10d + decl %r10d + jnz .Loop_sqrx_383 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 +.cfi_restore %r15 + movq 48(%rsp),%r14 +.cfi_restore %r14 + movq 56(%rsp),%r13 +.cfi_restore %r13 + movq 64(%rsp),%r12 +.cfi_restore %r12 + movq 72(%rsp),%rbx +.cfi_restore %rbx + movq 80(%rsp),%rbp +.cfi_restore %rbp + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -8*11 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_n_mul_mont_383,.-sqrx_n_mul_mont_383 +.type __mulx_mont_383_nonred,@function +.align 32 +__mulx_mont_383_nonred: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + movq %r8,%rax + imulq 8(%rsp),%r8 + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %r15,%rbp + adoxq %rbp,%r15 + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %rax,%r14 + adoxq %rax,%r15 + adcxq %rax,%r15 + movq %r9,%r8 + imulq 8(%rsp),%r9 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rax,%rbp + adoxq %rbp,%rax + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r10,%r9 + imulq 8(%rsp),%r10 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %r8,%rbp + adoxq %rbp,%r8 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r11,%r10 + imulq 8(%rsp),%r11 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r9 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r12,%r11 + imulq 8(%rsp),%r12 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r10 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r14,%rdx + adcxq %rdi,%r9 + adoxq %rbp,%r10 + adcq $0,%r10 + movq %r8,%r12 + + movq %r14,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r9,%rdi + movq %r8,24(%rbx) + movq %r9,32(%rbx) + movq %r10,40(%rbx) + movq %r10,%rbp + + .byte 0xf3,0xc3 +.cfi_endproc +.size __mulx_mont_383_nonred,.-__mulx_mont_383_nonred +.globl sqrx_mont_382x +.hidden sqrx_mont_382x +.type sqrx_mont_382x,@function +.align 32 +sqrx_mont_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + adcq %r12,%r12 + adcq %rdi,%rdi + adcq %rbp,%rbp + + movq %rdx,48(%rbx) + movq %r15,56(%rbx) + movq %rax,64(%rbx) + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32-128(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + + + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + movq 32+96(%rsp),%r14 + leaq 128(%rcx),%rcx + movq 32+0(%rsp),%r8 + andq %r14,%r8 + movq 32+8(%rsp),%r9 + andq %r14,%r9 + movq 32+16(%rsp),%r10 + andq %r14,%r10 + movq 32+24(%rsp),%r11 + andq %r14,%r11 + movq 32+32(%rsp),%r13 + andq %r14,%r13 + andq 32+40(%rsp),%r14 + + subq %r8,%rdx + movq 0(%rcx),%r8 + sbbq %r9,%r15 + movq 8(%rcx),%r9 + sbbq %r10,%rax + movq 16(%rcx),%r10 + sbbq %r11,%r12 + movq 24(%rcx),%r11 + sbbq %r13,%rdi + movq 32(%rcx),%r13 + sbbq %r14,%rbp + sbbq %r14,%r14 + + andq %r14,%r8 + andq %r14,%r9 + andq %r14,%r10 + andq %r14,%r11 + andq %r14,%r13 + andq 40(%rcx),%r14 + + addq %r8,%rdx + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%r12 + adcq %r13,%rdi + adcq %r14,%rbp + + movq %rdx,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc +.size sqrx_mont_382x,.-sqrx_mont_382x + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/sha256-armv8.S b/crypto/blst_src/build/elf/sha256-armv8.S new file mode 100644 index 00000000000..7341decf4f5 --- /dev/null +++ b/crypto/blst_src/build/elf/sha256-armv8.S @@ -0,0 +1,1077 @@ +// +// Copyright Supranational LLC +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// ==================================================================== +// Written by Andy Polyakov, @dot-asm, initially for the OpenSSL +// project. +// ==================================================================== +// +// sha256_block procedure for ARMv8. +// +// This module is stripped of scalar code paths, with raionale that all +// known processors are NEON-capable. +// +// See original module at CRYPTOGAMS for further details. + +.text + +.align 6 +.type .LK256,%object +.LK256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0 //terminator +.size .LK256,.-.LK256 +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.align 2 +.align 2 +.globl blst_sha256_block_armv8 +.type blst_sha256_block_armv8,%function +.align 6 +blst_sha256_block_armv8: +.Lv8_entry: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ld1 {v0.4s,v1.4s},[x0] + adr x3,.LK256 + +.Loop_hw: + ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 + sub x2,x2,#1 + ld1 {v16.4s},[x3],#16 + rev32 v4.16b,v4.16b + rev32 v5.16b,v5.16b + rev32 v6.16b,v6.16b + rev32 v7.16b,v7.16b + orr v18.16b,v0.16b,v0.16b // offload + orr v19.16b,v1.16b,v1.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.inst 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + ld1 {v17.4s},[x3] + add v16.4s,v16.4s,v6.4s + sub x3,x3,#64*4-16 // rewind + orr v2.16b,v0.16b,v0.16b +.inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + add v17.4s,v17.4s,v7.4s + orr v2.16b,v0.16b,v0.16b +.inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + add v0.4s,v0.4s,v18.4s + add v1.4s,v1.4s,v19.4s + + cbnz x2,.Loop_hw + + st1 {v0.4s,v1.4s},[x0] + + ldr x29,[sp],#16 + ret +.size blst_sha256_block_armv8,.-blst_sha256_block_armv8 +.globl blst_sha256_block_data_order +.type blst_sha256_block_data_order,%function +.align 4 +blst_sha256_block_data_order: + stp x29, x30, [sp, #-16]! + mov x29, sp + sub sp,sp,#16*4 + + adr x16,.LK256 + add x2,x1,x2,lsl#6 // len to point at the end of inp + + ld1 {v0.16b},[x1], #16 + ld1 {v1.16b},[x1], #16 + ld1 {v2.16b},[x1], #16 + ld1 {v3.16b},[x1], #16 + ld1 {v4.4s},[x16], #16 + ld1 {v5.4s},[x16], #16 + ld1 {v6.4s},[x16], #16 + ld1 {v7.4s},[x16], #16 + rev32 v0.16b,v0.16b // yes, even on + rev32 v1.16b,v1.16b // big-endian + rev32 v2.16b,v2.16b + rev32 v3.16b,v3.16b + mov x17,sp + add v4.4s,v4.4s,v0.4s + add v5.4s,v5.4s,v1.4s + add v6.4s,v6.4s,v2.4s + st1 {v4.4s,v5.4s},[x17], #32 + add v7.4s,v7.4s,v3.4s + st1 {v6.4s,v7.4s},[x17] + sub x17,x17,#32 + + ldp w3,w4,[x0] + ldp w5,w6,[x0,#8] + ldp w7,w8,[x0,#16] + ldp w9,w10,[x0,#24] + ldr w12,[sp,#0] + mov w13,wzr + eor w14,w4,w5 + mov w15,wzr + b .L_00_48 + +.align 4 +.L_00_48: + ext v4.16b,v0.16b,v1.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v2.16b,v3.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v3.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v0.4s,v0.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v0.4s,v0.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v0.4s,v0.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v0.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v0.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v0.4s,#15 + add w8,w8,w12 + ushr v17.4s,v0.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v0.4s,#13 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v0.4s,v0.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v0.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v1.16b,v2.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v3.16b,v0.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v0.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v1.4s,v1.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v1.4s,v1.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v1.4s,v1.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v1.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v1.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v1.4s,#15 + add w4,w4,w12 + ushr v17.4s,v1.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v1.4s,#13 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v1.4s,v1.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v1.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + ext v4.16b,v2.16b,v3.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v0.16b,v1.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v1.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v2.4s,v2.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v2.4s,v2.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v2.4s,v2.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v2.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v2.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v2.4s,#15 + add w8,w8,w12 + ushr v17.4s,v2.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v2.4s,#13 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v2.4s,v2.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v2.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v3.16b,v0.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v1.16b,v2.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v2.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v3.4s,v3.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v3.4s,v3.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v3.4s,v3.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v3.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v3.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v3.4s,#15 + add w4,w4,w12 + ushr v17.4s,v3.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v3.4s,#13 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v3.4s,v3.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v3.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[x16] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + cmp w12,#0 // check for K256 terminator + ldr w12,[sp,#0] + sub x17,x17,#64 + bne .L_00_48 + + sub x16,x16,#256 // rewind x16 + cmp x1,x2 + mov x17, #64 + csel x17, x17, xzr, eq + sub x1,x1,x17 // avoid SEGV + mov x17,sp + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v0.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v0.16b,v0.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v0.4s + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v1.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v1.16b,v1.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v1.4s + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v2.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v2.16b,v2.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v2.4s + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v3.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v3.16b,v3.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v3.4s + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w3,w3,w15 // h+=Sigma0(a) from the past + ldp w11,w12,[x0,#0] + add w3,w3,w13 // h+=Maj(a,b,c) from the past + ldp w13,w14,[x0,#8] + add w3,w3,w11 // accumulate + add w4,w4,w12 + ldp w11,w12,[x0,#16] + add w5,w5,w13 + add w6,w6,w14 + ldp w13,w14,[x0,#24] + add w7,w7,w11 + add w8,w8,w12 + ldr w12,[sp,#0] + stp w3,w4,[x0,#0] + add w9,w9,w13 + mov w13,wzr + stp w5,w6,[x0,#8] + add w10,w10,w14 + stp w7,w8,[x0,#16] + eor w14,w4,w5 + stp w9,w10,[x0,#24] + mov w15,wzr + mov x17,sp + b.ne .L_00_48 + + ldr x29,[x29] + add sp,sp,#16*4+16 + ret +.size blst_sha256_block_data_order,.-blst_sha256_block_data_order +.globl blst_sha256_emit +.hidden blst_sha256_emit +.type blst_sha256_emit,%function +.align 4 +blst_sha256_emit: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] +#ifndef __AARCH64EB__ + rev x4,x4 + rev x5,x5 + rev x6,x6 + rev x7,x7 +#endif + str w4,[x0,#4] + lsr x4,x4,#32 + str w5,[x0,#12] + lsr x5,x5,#32 + str w6,[x0,#20] + lsr x6,x6,#32 + str w7,[x0,#28] + lsr x7,x7,#32 + str w4,[x0,#0] + str w5,[x0,#8] + str w6,[x0,#16] + str w7,[x0,#24] + ret +.size blst_sha256_emit,.-blst_sha256_emit + +.globl blst_sha256_bcopy +.hidden blst_sha256_bcopy +.type blst_sha256_bcopy,%function +.align 4 +blst_sha256_bcopy: +.Loop_bcopy: + ldrb w3,[x1],#1 + sub x2,x2,#1 + strb w3,[x0],#1 + cbnz x2,.Loop_bcopy + ret +.size blst_sha256_bcopy,.-blst_sha256_bcopy + +.globl blst_sha256_hcopy +.hidden blst_sha256_hcopy +.type blst_sha256_hcopy,%function +.align 4 +blst_sha256_hcopy: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + stp x4,x5,[x0] + stp x6,x7,[x0,#16] + ret +.size blst_sha256_hcopy,.-blst_sha256_hcopy diff --git a/crypto/blst_src/build/elf/sha256-portable-x86_64.s b/crypto/blst_src/build/elf/sha256-portable-x86_64.s new file mode 100644 index 00000000000..20b5c411306 --- /dev/null +++ b/crypto/blst_src/build/elf/sha256-portable-x86_64.s @@ -0,0 +1,1754 @@ +.text + +.globl blst_sha256_block_data_order +.type blst_sha256_block_data_order,@function +.align 16 +blst_sha256_block_data_order: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-16 + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + shlq $4,%rdx + subq $64+24,%rsp +.cfi_adjust_cfa_offset 16*4+3*8 + leaq (%rsi,%rdx,4),%rdx + movq %rdi,64+0(%rsp) + movq %rsi,64+8(%rsp) + movq %rdx,64+16(%rsp) + + + movl 0(%rdi),%eax + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + jmp .Lloop + +.align 16 +.Lloop: + movl %ebx,%edi + leaq K256(%rip),%rbp + xorl %ecx,%edi + movl 0(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 0(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 4(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 4(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 8(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 8(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 12(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 12(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 16(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 16(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 20(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 20(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 24(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 24(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 28(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 28(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + addl %r14d,%eax + movl 32(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 32(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 36(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 36(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 40(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 40(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 44(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 44(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 48(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 48(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 52(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 52(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 56(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 56(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 60(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 60(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + jmp .Lrounds_16_xx +.align 16 +.Lrounds_16_xx: + movl 4(%rsp),%r13d + movl 56(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 36(%rsp),%r12d + + addl 0(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 64(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 8(%rsp),%r13d + movl 60(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 40(%rsp),%r12d + + addl 4(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 68(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 12(%rsp),%r13d + movl 0(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 44(%rsp),%r12d + + addl 8(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 72(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 16(%rsp),%r13d + movl 4(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 48(%rsp),%r12d + + addl 12(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 76(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 20(%rsp),%r13d + movl 8(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 52(%rsp),%r12d + + addl 16(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 80(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 24(%rsp),%r13d + movl 12(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 56(%rsp),%r12d + + addl 20(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 84(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 28(%rsp),%r13d + movl 16(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 60(%rsp),%r12d + + addl 24(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 88(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 32(%rsp),%r13d + movl 20(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 0(%rsp),%r12d + + addl 28(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 92(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + movl 36(%rsp),%r13d + movl 24(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 4(%rsp),%r12d + + addl 32(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 96(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 40(%rsp),%r13d + movl 28(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 8(%rsp),%r12d + + addl 36(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 100(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 44(%rsp),%r13d + movl 32(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 12(%rsp),%r12d + + addl 40(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 104(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 48(%rsp),%r13d + movl 36(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 16(%rsp),%r12d + + addl 44(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 108(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 52(%rsp),%r13d + movl 40(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 20(%rsp),%r12d + + addl 48(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 112(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 56(%rsp),%r13d + movl 44(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 24(%rsp),%r12d + + addl 52(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 116(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 60(%rsp),%r13d + movl 48(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 28(%rsp),%r12d + + addl 56(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 120(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 0(%rsp),%r13d + movl 52(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 32(%rsp),%r12d + + addl 60(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 124(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + leaq 64(%rbp),%rbp + cmpb $0x19,3(%rbp) + jnz .Lrounds_16_xx + + movq 64+0(%rsp),%rdi + addl %r14d,%eax + leaq 64(%rsi),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + cmpq 64+16(%rsp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb .Lloop + + leaq 64+24+48(%rsp),%r11 +.cfi_def_cfa %r11,8 + movq 64+24(%rsp),%r15 +.cfi_restore %r15 + movq -40(%r11),%r14 +.cfi_restore %r14 + movq -32(%r11),%r13 +.cfi_restore %r13 + movq -24(%r11),%r12 +.cfi_restore %r12 + movq -16(%r11),%rbp +.cfi_restore %rbp + movq -8(%r11),%rbx +.cfi_restore %rbx + + leaq (%r11),%rsp + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_block_data_order,.-blst_sha256_block_data_order + +.align 64 +.type K256,@object +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl blst_sha256_emit +.hidden blst_sha256_emit +.type blst_sha256_emit,@function +.align 16 +blst_sha256_emit: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + bswapq %r8 + movq 24(%rsi),%r11 + bswapq %r9 + movl %r8d,4(%rdi) + bswapq %r10 + movl %r9d,12(%rdi) + bswapq %r11 + movl %r10d,20(%rdi) + shrq $32,%r8 + movl %r11d,28(%rdi) + shrq $32,%r9 + movl %r8d,0(%rdi) + shrq $32,%r10 + movl %r9d,8(%rdi) + shrq $32,%r11 + movl %r10d,16(%rdi) + movl %r11d,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_emit,.-blst_sha256_emit + +.globl blst_sha256_bcopy +.hidden blst_sha256_bcopy +.type blst_sha256_bcopy,@function +.align 16 +blst_sha256_bcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rsi,%rdi +.Loop_bcopy: + movzbl (%rsi),%eax + leaq 1(%rsi),%rsi + movb %al,-1(%rdi,%rsi,1) + decq %rdx + jnz .Loop_bcopy + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_bcopy,.-blst_sha256_bcopy + +.globl blst_sha256_hcopy +.hidden blst_sha256_hcopy +.type blst_sha256_hcopy,@function +.align 16 +blst_sha256_hcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_hcopy,.-blst_sha256_hcopy + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/elf/sha256-x86_64.s b/crypto/blst_src/build/elf/sha256-x86_64.s new file mode 100644 index 00000000000..47fdc5bc57a --- /dev/null +++ b/crypto/blst_src/build/elf/sha256-x86_64.s @@ -0,0 +1,1446 @@ +.text + +.align 64 +.type K256,@object +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f +.long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff +.long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl blst_sha256_block_data_order_shaext +.hidden blst_sha256_block_data_order_shaext +.type blst_sha256_block_data_order_shaext,@function +.align 64 +blst_sha256_block_data_order_shaext: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + leaq K256+128(%rip),%rcx + movdqu (%rdi),%xmm1 + movdqu 16(%rdi),%xmm2 + movdqa 256-128(%rcx),%xmm7 + + pshufd $0x1b,%xmm1,%xmm0 + pshufd $0xb1,%xmm1,%xmm1 + pshufd $0x1b,%xmm2,%xmm2 + movdqa %xmm7,%xmm8 +.byte 102,15,58,15,202,8 + punpcklqdq %xmm0,%xmm2 + jmp .Loop_shaext + +.align 16 +.Loop_shaext: + movdqu (%rsi),%xmm3 + movdqu 16(%rsi),%xmm4 + movdqu 32(%rsi),%xmm5 +.byte 102,15,56,0,223 + movdqu 48(%rsi),%xmm6 + + movdqa 0-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 102,15,56,0,231 + movdqa %xmm2,%xmm10 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + nop + movdqa %xmm1,%xmm9 +.byte 15,56,203,202 + + movdqa 16-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 102,15,56,0,239 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + leaq 64(%rsi),%rsi +.byte 15,56,204,220 +.byte 15,56,203,202 + + movdqa 32-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 102,15,56,0,247 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + + movdqa 48-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 64-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 80-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 96-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 112-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 128-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 144-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 160-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 176-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 192-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 208-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 +.byte 15,56,203,202 + paddd %xmm7,%xmm6 + + movdqa 224-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 +.byte 15,56,205,245 + movdqa %xmm8,%xmm7 +.byte 15,56,203,202 + + movdqa 240-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 + nop +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + decq %rdx + nop +.byte 15,56,203,202 + + paddd %xmm10,%xmm2 + paddd %xmm9,%xmm1 + jnz .Loop_shaext + + pshufd $0xb1,%xmm2,%xmm2 + pshufd $0x1b,%xmm1,%xmm7 + pshufd $0xb1,%xmm1,%xmm1 + punpckhqdq %xmm2,%xmm1 +.byte 102,15,58,15,215,8 + + movdqu %xmm1,(%rdi) + movdqu %xmm2,16(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_block_data_order_shaext,.-blst_sha256_block_data_order_shaext +.globl blst_sha256_block_data_order +.hidden blst_sha256_block_data_order +.type blst_sha256_block_data_order,@function +.align 64 +blst_sha256_block_data_order: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + shlq $4,%rdx + subq $40,%rsp +.cfi_adjust_cfa_offset 40 + leaq (%rsi,%rdx,4),%rdx + movq %rdi,0(%rsp) + + movq %rdx,16(%rsp) + movq %rsp,%rbp +.cfi_def_cfa_register %rbp + + + leaq -64(%rsp),%rsp + movl 0(%rdi),%eax + andq $-64,%rsp + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + + + jmp .Lloop_ssse3 +.align 16 +.Lloop_ssse3: + movdqa K256+256(%rip),%xmm7 + movq %rsi,8(%rbp) + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 +.byte 102,15,56,0,199 + movdqu 48(%rsi),%xmm3 + leaq K256(%rip),%rsi +.byte 102,15,56,0,207 + movdqa 0(%rsi),%xmm4 + movdqa 16(%rsi),%xmm5 +.byte 102,15,56,0,215 + paddd %xmm0,%xmm4 + movdqa 32(%rsi),%xmm6 +.byte 102,15,56,0,223 + movdqa 48(%rsi),%xmm7 + paddd %xmm1,%xmm5 + paddd %xmm2,%xmm6 + paddd %xmm3,%xmm7 + movdqa %xmm4,0(%rsp) + movl %eax,%r14d + movdqa %xmm5,16(%rsp) + movl %ebx,%edi + movdqa %xmm6,32(%rsp) + xorl %ecx,%edi + movdqa %xmm7,48(%rsp) + movl %r8d,%r13d + jmp .Lssse3_00_47 + +.align 16 +.Lssse3_00_47: + subq $-64,%rsi + rorl $14,%r13d + movdqa %xmm1,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm3,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,224,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,250,4 + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm3,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 4(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm0 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm0 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm0,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 0(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm0,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,0(%rsp) + rorl $14,%r13d + movdqa %xmm2,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm0,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,225,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,251,4 + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm0,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 20(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm1 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm1 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm1,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 16(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm1,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,16(%rsp) + rorl $14,%r13d + movdqa %xmm3,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm1,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,226,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,248,4 + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm1,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 36(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm2 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm2 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm2,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 32(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm2,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,32(%rsp) + rorl $14,%r13d + movdqa %xmm0,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm2,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,227,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,249,4 + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm2,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 52(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm3 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm3 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm3,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 48(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm3,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,48(%rsp) + cmpb $0,67(%rsi) + jne .Lssse3_00_47 + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 4(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 20(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 36(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 52(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + movq 0(%rbp),%rdi + movl %r14d,%eax + movq 8(%rbp),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + leaq 64(%rsi),%rsi + cmpq 16(%rbp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb .Lloop_ssse3 + + xorps %xmm0,%xmm0 + leaq 40+48(%rbp),%r11 +.cfi_def_cfa %r11,8 + movaps %xmm0,0(%rsp) + movaps %xmm0,16(%rsp) + movaps %xmm0,32(%rsp) + movaps %xmm0,48(%rsp) + movq 40(%rbp),%r15 +.cfi_restore %r15 + movq -40(%r11),%r14 +.cfi_restore %r14 + movq -32(%r11),%r13 +.cfi_restore %r13 + movq -24(%r11),%r12 +.cfi_restore %r12 + movq -16(%r11),%rbx +.cfi_restore %rbx + movq -8(%r11),%rbp +.cfi_restore %rbp + + leaq (%r11),%rsp + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_block_data_order,.-blst_sha256_block_data_order +.globl blst_sha256_emit +.hidden blst_sha256_emit +.type blst_sha256_emit,@function +.align 16 +blst_sha256_emit: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + bswapq %r8 + movq 24(%rsi),%r11 + bswapq %r9 + movl %r8d,4(%rdi) + bswapq %r10 + movl %r9d,12(%rdi) + bswapq %r11 + movl %r10d,20(%rdi) + shrq $32,%r8 + movl %r11d,28(%rdi) + shrq $32,%r9 + movl %r8d,0(%rdi) + shrq $32,%r10 + movl %r9d,8(%rdi) + shrq $32,%r11 + movl %r10d,16(%rdi) + movl %r11d,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_emit,.-blst_sha256_emit + +.globl blst_sha256_bcopy +.hidden blst_sha256_bcopy +.type blst_sha256_bcopy,@function +.align 16 +blst_sha256_bcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rsi,%rdi +.Loop_bcopy: + movzbl (%rsi),%eax + leaq 1(%rsi),%rsi + movb %al,-1(%rdi,%rsi,1) + decq %rdx + jnz .Loop_bcopy + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_bcopy,.-blst_sha256_bcopy + +.globl blst_sha256_hcopy +.hidden blst_sha256_hcopy +.type blst_sha256_hcopy,@function +.align 16 +blst_sha256_hcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc +.size blst_sha256_hcopy,.-blst_sha256_hcopy + +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000002,4,3 +.align 8 +2: diff --git a/crypto/blst_src/build/mach-o/add_mod_256-armv8.S b/crypto/blst_src/build/mach-o/add_mod_256-armv8.S new file mode 100644 index 00000000000..198d65aef69 --- /dev/null +++ b/crypto/blst_src/build/mach-o/add_mod_256-armv8.S @@ -0,0 +1,379 @@ +.text + +.globl _add_mod_256 +.private_extern _add_mod_256 + +.align 5 +_add_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + adds x8,x8,x12 + ldp x14,x15,[x2,#16] + adcs x9,x9,x13 + ldp x4,x5,[x3] + adcs x10,x10,x14 + ldp x6,x7,[x3,#16] + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret + + +.globl _mul_by_3_mod_256 +.private_extern _mul_by_3_mod_256 + +.align 5 +_mul_by_3_mod_256: + ldp x12,x13,[x1] + ldp x14,x15,[x1,#16] + + adds x8,x12,x12 + ldp x4,x5,[x2] + adcs x9,x13,x13 + ldp x6,x7,[x2,#16] + adcs x10,x14,x14 + adcs x11,x15,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + adds x8,x8,x12 + adcs x9,x9,x13 + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + stp x8,x9,[x0] + csel x11,x11,x2,lo + stp x10,x11,[x0,#16] + + ret + + +.globl _lshift_mod_256 +.private_extern _lshift_mod_256 + +.align 5 +_lshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +Loop_lshift_mod_256: + adds x8,x8,x8 + sub x2,x2,#1 + adcs x9,x9,x9 + adcs x10,x10,x10 + adcs x11,x11,x11 + adc x3,xzr,xzr + + subs x12,x8,x4 + sbcs x13,x9,x5 + sbcs x14,x10,x6 + sbcs x15,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x12,lo + csel x9,x9,x13,lo + csel x10,x10,x14,lo + csel x11,x11,x15,lo + + cbnz x2,Loop_lshift_mod_256 + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + + +.globl _rshift_mod_256 +.private_extern _rshift_mod_256 + +.align 5 +_rshift_mod_256: + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +Loop_rshift: + adds x12,x8,x4 + sub x2,x2,#1 + adcs x13,x9,x5 + adcs x14,x10,x6 + adcs x15,x11,x7 + adc x3,xzr,xzr + tst x8,#1 + + csel x12,x12,x8,ne + csel x13,x13,x9,ne + csel x14,x14,x10,ne + csel x15,x15,x11,ne + csel x3,x3,xzr,ne + + extr x8,x13,x12,#1 + extr x9,x14,x13,#1 + extr x10,x15,x14,#1 + extr x11,x3,x15,#1 + + cbnz x2,Loop_rshift + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + + +.globl _cneg_mod_256 +.private_extern _cneg_mod_256 + +.align 5 +_cneg_mod_256: + ldp x8,x9,[x1] + ldp x4,x5,[x3] + + ldp x10,x11,[x1,#16] + subs x12,x4,x8 + ldp x6,x7,[x3,#16] + orr x4,x8,x9 + sbcs x13,x5,x9 + orr x5,x10,x11 + sbcs x14,x6,x10 + orr x3,x4,x5 + sbc x15,x7,x11 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x8,x8,x12,eq + csel x9,x9,x13,eq + csel x10,x10,x14,eq + stp x8,x9,[x0] + csel x11,x11,x15,eq + stp x10,x11,[x0,#16] + + ret + + +.globl _sub_mod_256 +.private_extern _sub_mod_256 + +.align 5 +_sub_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + subs x8,x8,x12 + ldp x14,x15,[x2,#16] + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + stp x8,x9,[x0] + adc x11,x11,x7 + stp x10,x11,[x0,#16] + + ret + + +.globl _check_mod_256 +.private_extern _check_mod_256 + +.align 5 +_check_mod_256: + ldp x8,x9,[x0] + ldp x10,x11,[x0,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + subs xzr,x8,x4 + sbcs xzr,x9,x5 + orr x8,x8,x9 + sbcs xzr,x10,x6 + orr x8,x8,x10 + sbcs xzr,x11,x7 + orr x8,x8,x11 + sbc x1,xzr,xzr + + cmp x8,#0 + mov x0,#1 + csel x0,x0,xzr,ne + and x0,x0,x1 + + ret + + +.globl _add_n_check_mod_256 +.private_extern _add_n_check_mod_256 + +.align 5 +_add_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + adds x8,x8,x12 + ldp x4,x5,[x3] + adcs x9,x9,x13 + ldp x6,x7,[x3,#16] + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csel x8,x8,x16,lo + csel x9,x9,x17,lo + csel x10,x10,x1,lo + csel x11,x11,x2,lo + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret + + +.globl _sub_n_check_mod_256 +.private_extern _sub_n_check_mod_256 + +.align 5 +_sub_n_check_mod_256: + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + subs x8,x8,x12 + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + adc x11,x11,x7 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + csel x0, x17, xzr, ne + + ret + diff --git a/crypto/blst_src/build/mach-o/add_mod_256-x86_64.s b/crypto/blst_src/build/mach-o/add_mod_256-x86_64.s new file mode 100644 index 00000000000..19e5ba9834f --- /dev/null +++ b/crypto/blst_src/build/mach-o/add_mod_256-x86_64.s @@ -0,0 +1,564 @@ +.text + +.globl _add_mod_256 +.private_extern _add_mod_256 + +.p2align 5 +_add_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +L$oaded_a_add_mod_256: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _mul_by_3_mod_256 +.private_extern _mul_by_3_mod_256 + +.p2align 5 +_mul_by_3_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq %rdx,%rcx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rsi,%rdx + movq 24(%rsi),%r11 + + call __lshift_mod_256 + movq 0(%rsp),%r12 +.cfi_restore %r12 + jmp L$oaded_a_add_mod_256 + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__lshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + movq %r8,%rax + adcq %r10,%r10 + movq %r9,%rsi + adcq %r11,%r11 + sbbq %r12,%r12 + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + cmovcq %rbx,%r10 + cmovcq %rbp,%r11 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _lshift_mod_256 +.private_extern _lshift_mod_256 + +.p2align 5 +_lshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +L$oop_lshift_mod_256: + call __lshift_mod_256 + decl %edx + jnz L$oop_lshift_mod_256 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _rshift_mod_256 +.private_extern _rshift_mod_256 + +.p2align 5 +_rshift_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rbp + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + +L$oop_rshift_mod_256: + movq %rbp,%r8 + andq $1,%rbp + movq 0(%rcx),%rax + negq %rbp + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + + andq %rbp,%rax + andq %rbp,%rsi + andq %rbp,%rbx + andq 24(%rcx),%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + adcq %rbx,%r10 + adcq %rbp,%r11 + sbbq %rax,%rax + + shrq $1,%r8 + movq %r9,%rbp + shrq $1,%r9 + movq %r10,%rbx + shrq $1,%r10 + movq %r11,%rsi + shrq $1,%r11 + + shlq $63,%rbp + shlq $63,%rbx + orq %r8,%rbp + shlq $63,%rsi + orq %rbx,%r9 + shlq $63,%rax + orq %rsi,%r10 + orq %rax,%r11 + + decl %edx + jnz L$oop_rshift_mod_256 + + movq %rbp,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _cneg_mod_256 +.private_extern _cneg_mod_256 + +.p2align 5 +_cneg_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq 0(%rsi),%r12 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %r12,%r8 + movq 24(%rsi),%r11 + orq %r9,%r12 + orq %r10,%r12 + orq %r11,%r12 + movq $-1,%rbp + + movq 0(%rcx),%rax + cmovnzq %rbp,%r12 + movq 8(%rcx),%rsi + movq 16(%rcx),%rbx + andq %r12,%rax + movq 24(%rcx),%rbp + andq %r12,%rsi + andq %r12,%rbx + andq %r12,%rbp + + subq %r8,%rax + sbbq %r9,%rsi + sbbq %r10,%rbx + sbbq %r11,%rbp + + orq %rdx,%rdx + + cmovzq %r8,%rax + cmovzq %r9,%rsi + movq %rax,0(%rdi) + cmovzq %r10,%rbx + movq %rsi,8(%rdi) + cmovzq %r11,%rbp + movq %rbx,16(%rdi) + movq %rbp,24(%rdi) + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _sub_mod_256 +.private_extern _sub_mod_256 + +.p2align 5 +_sub_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _check_mod_256 +.private_extern _check_mod_256 + +.p2align 5 +_check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + movq 0(%rdi),%rax + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + + movq %rax,%r8 + orq %r9,%rax + orq %r10,%rax + orq %r11,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq %rsi,%rsi + + movq $1,%rdx + cmpq $0,%rax + cmovneq %rdx,%rax + andq %rsi,%rax + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _add_n_check_mod_256 +.private_extern _add_n_check_mod_256 + +.p2align 5 +_add_n_check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + movq %r8,%rax + adcq 16(%rdx),%r10 + movq %r9,%rsi + adcq 24(%rdx),%r11 + sbbq %rdx,%rdx + + movq %r10,%rbx + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + sbbq 16(%rcx),%r10 + movq %r11,%rbp + sbbq 24(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %rax,%r8 + cmovcq %rsi,%r9 + movq %r8,0(%rdi) + cmovcq %rbx,%r10 + movq %r9,8(%rdi) + cmovcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _sub_n_check_mod_256 +.private_extern _sub_n_check_mod_256 + +.p2align 5 +_sub_n_check_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + subq 0(%rdx),%r8 + movq 0(%rcx),%rax + sbbq 8(%rdx),%r9 + movq 8(%rcx),%rsi + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rbx + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbp + sbbq %rdx,%rdx + + andq %rdx,%rax + andq %rdx,%rsi + andq %rdx,%rbx + andq %rdx,%rbp + + addq %rax,%r8 + adcq %rsi,%r9 + movq %r8,0(%rdi) + adcq %rbx,%r10 + movq %r9,8(%rdi) + adcq %rbp,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + orq %r9,%r8 + orq %r11,%r10 + orq %r10,%r8 + movq $1,%rax + cmovzq %r8,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/add_mod_384-armv8.S b/crypto/blst_src/build/mach-o/add_mod_384-armv8.S new file mode 100644 index 00000000000..a62995f2bed --- /dev/null +++ b/crypto/blst_src/build/mach-o/add_mod_384-armv8.S @@ -0,0 +1,1000 @@ +.text + +.globl _add_mod_384 +.private_extern _add_mod_384 + +.align 5 +_add_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + + +.align 5 +__add_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + +__add_mod_384_ab_are_loaded: + adds x10,x10,x16 + adcs x11,x11,x17 + adcs x12,x12,x19 + adcs x13,x13,x20 + adcs x14,x14,x21 + adcs x15,x15,x22 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret + + +.globl _add_mod_384x +.private_extern _add_mod_384x + +.align 5 +_add_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _rshift_mod_384 +.private_extern _rshift_mod_384 + +.align 5 +_rshift_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +Loop_rshift_mod_384: + sub x2,x2,#1 + bl __rshift_mod_384 + cbnz x2,Loop_rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + + +.align 5 +__rshift_mod_384: + sbfx x22,x10,#0,#1 + and x16,x22,x4 + and x17,x22,x5 + adds x10,x10,x16 + and x19,x22,x6 + adcs x11,x11,x17 + and x20,x22,x7 + adcs x12,x12,x19 + and x21,x22,x8 + adcs x13,x13,x20 + and x22,x22,x9 + adcs x14,x14,x21 + extr x10,x11,x10,#1 // a[0:5] >>= 1 + adcs x15,x15,x22 + extr x11,x12,x11,#1 + adc x22,xzr,xzr + extr x12,x13,x12,#1 + extr x13,x14,x13,#1 + extr x14,x15,x14,#1 + extr x15,x22,x15,#1 + ret + + +.globl _div_by_2_mod_384 +.private_extern _div_by_2_mod_384 + +.align 5 +_div_by_2_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _lshift_mod_384 +.private_extern _lshift_mod_384 + +.align 5 +_lshift_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +Loop_lshift_mod_384: + sub x2,x2,#1 + bl __lshift_mod_384 + cbnz x2,Loop_lshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + + +.align 5 +__lshift_mod_384: + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csel x10,x10,x16,lo + csel x11,x11,x17,lo + csel x12,x12,x19,lo + csel x13,x13,x20,lo + csel x14,x14,x21,lo + csel x15,x15,x22,lo + + ret + + +.globl _mul_by_3_mod_384 +.private_extern _mul_by_3_mod_384 + +.align 5 +_mul_by_3_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _mul_by_8_mod_384 +.private_extern _mul_by_8_mod_384 + +.align 5 +_mul_by_8_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _mul_by_3_mod_384x +.private_extern _mul_by_3_mod_384x + +.align 5 +_mul_by_3_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + + ldp x16,x17,[x1,#48] + ldp x19,x20,[x1,#64] + ldp x21,x22,[x1,#80] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _mul_by_8_mod_384x +.private_extern _mul_by_8_mod_384x + +.align 5 +_mul_by_8_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _cneg_mod_384 +.private_extern _cneg_mod_384 + +.align 5 +_cneg_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x4,x5,[x3] + ldp x12,x13,[x1,#16] + ldp x6,x7,[x3,#16] + + subs x16,x4,x10 + ldp x14,x15,[x1,#32] + ldp x8,x9,[x3,#32] + orr x3,x10,x11 + sbcs x17,x5,x11 + orr x3,x3,x12 + sbcs x19,x6,x12 + orr x3,x3,x13 + sbcs x20,x7,x13 + orr x3,x3,x14 + sbcs x21,x8,x14 + orr x3,x3,x15 + sbc x22,x9,x15 + + cmp x3,#0 + csetm x3,ne + ands x2,x2,x3 + + csel x10,x10,x16,eq + csel x11,x11,x17,eq + csel x12,x12,x19,eq + csel x13,x13,x20,eq + stp x10,x11,[x0] + csel x14,x14,x21,eq + stp x12,x13,[x0,#16] + csel x15,x15,x22,eq + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _sub_mod_384 +.private_extern _sub_mod_384 + +.align 5 +_sub_mod_384: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + + +.align 5 +__sub_mod_384: + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + + subs x10,x10,x16 + sbcs x11,x11,x17 + sbcs x12,x12,x19 + sbcs x13,x13,x20 + sbcs x14,x14,x21 + sbcs x15,x15,x22 + sbc x3,xzr,xzr + + and x16,x4,x3 + and x17,x5,x3 + adds x10,x10,x16 + and x19,x6,x3 + adcs x11,x11,x17 + and x20,x7,x3 + adcs x12,x12,x19 + and x21,x8,x3 + adcs x13,x13,x20 + and x22,x9,x3 + adcs x14,x14,x21 + adc x15,x15,x22 + + ret + + +.globl _sub_mod_384x +.private_extern _sub_mod_384x + +.align 5 +_sub_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _mul_by_1_plus_i_mod_384x +.private_extern _mul_by_1_plus_i_mod_384x + +.align 5 +_mul_by_1_plus_i_mod_384x: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + add x2,x1,#48 + + bl __sub_mod_384 // a->re - a->im + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __add_mod_384_ab_are_loaded // a->re + a->im + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + + +.globl _sgn0_pty_mod_384 +.private_extern _sgn0_pty_mod_384 + +.align 5 +_sgn0_pty_mod_384: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x0,x10,#1 + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x3,x3,xzr + + mvn x3,x3 + and x3,x3,#2 + orr x0,x0,x3 + + ret + + +.globl _sgn0_pty_mod_384x +.private_extern _sgn0_pty_mod_384x + +.align 5 +_sgn0_pty_mod_384x: + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x2,x10,#1 + orr x3,x10,x11 + adds x10,x10,x10 + orr x3,x3,x12 + adcs x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + ldp x10,x11,[x0,#48] + ldp x12,x13,[x0,#64] + ldp x14,x15,[x0,#80] + + mvn x16,x16 + and x16,x16,#2 + orr x2,x2,x16 + + and x0,x10,#1 + orr x1,x10,x11 + adds x10,x10,x10 + orr x1,x1,x12 + adcs x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + mvn x16,x16 + and x16,x16,#2 + orr x0,x0,x16 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ret + +.globl _vec_select_32 +.private_extern _vec_select_32 + +.align 5 +_vec_select_32: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl _vec_select_48 +.private_extern _vec_select_48 + +.align 5 +_vec_select_48: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl _vec_select_96 +.private_extern _vec_select_96 + +.align 5 +_vec_select_96: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl _vec_select_192 +.private_extern _vec_select_192 + +.align 5 +_vec_select_192: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl _vec_select_144 +.private_extern _vec_select_144 + +.align 5 +_vec_select_144: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + +.globl _vec_select_288 +.private_extern _vec_select_288 + +.align 5 +_vec_select_288: + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + +.globl _vec_prefetch +.private_extern _vec_prefetch + +.align 5 +_vec_prefetch: + add x1, x1, x0 + sub x1, x1, #1 + mov x2, #64 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + csel x2, xzr, x2, hi + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + csel x0, x1, x0, hi + prfm pldl1keep, [x0] + ret + +.globl _vec_is_zero_16x +.private_extern _vec_is_zero_16x + +.align 5 +_vec_is_zero_16x: + ld1 {v0.2d}, [x0], #16 + lsr x1, x1, #4 + sub x1, x1, #1 + cbz x1, Loop_is_zero_done + +Loop_is_zero: + ld1 {v1.2d}, [x0], #16 + orr v0.16b, v0.16b, v1.16b + sub x1, x1, #1 + cbnz x1, Loop_is_zero + +Loop_is_zero_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret + +.globl _vec_is_equal_16x +.private_extern _vec_is_equal_16x + +.align 5 +_vec_is_equal_16x: + ld1 {v0.2d}, [x0], #16 + ld1 {v1.2d}, [x1], #16 + lsr x2, x2, #4 + eor v0.16b, v0.16b, v1.16b + +Loop_is_equal: + sub x2, x2, #1 + cbz x2, Loop_is_equal_done + ld1 {v1.2d}, [x0], #16 + ld1 {v2.2d}, [x1], #16 + eor v1.16b, v1.16b, v2.16b + orr v0.16b, v0.16b, v1.16b + b Loop_is_equal + nop + +Loop_is_equal_done: + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + csel x0, x0, xzr, eq + ret + diff --git a/crypto/blst_src/build/mach-o/add_mod_384-x86_64.s b/crypto/blst_src/build/mach-o/add_mod_384-x86_64.s new file mode 100644 index 00000000000..974978e3425 --- /dev/null +++ b/crypto/blst_src/build/mach-o/add_mod_384-x86_64.s @@ -0,0 +1,1899 @@ +.text + +.globl _add_mod_384 +.private_extern _add_mod_384 + +.p2align 5 +_add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __add_mod_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__add_mod_384_a_is_loaded: + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _add_mod_384x +.private_extern _add_mod_384x + +.p2align 5 +_add_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 24 + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __add_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + movq 24+0(%rsp),%r15 +.cfi_restore %r15 + movq 24+8(%rsp),%r14 +.cfi_restore %r14 + movq 24+16(%rsp),%r13 +.cfi_restore %r13 + movq 24+24(%rsp),%r12 +.cfi_restore %r12 + movq 24+32(%rsp),%rbx +.cfi_restore %rbx + movq 24+40(%rsp),%rbp +.cfi_restore %rbp + leaq 24+48(%rsp),%rsp +.cfi_adjust_cfa_offset -24-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _rshift_mod_384 +.private_extern _rshift_mod_384 + +.p2align 5 +_rshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +L$oop_rshift_mod_384: + call __rshift_mod_384 + decl %edx + jnz L$oop_rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__rshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rsi + movq 0(%rcx),%r14 + andq %r8,%rsi + movq 8(%rcx),%r15 + negq %rsi + movq 16(%rcx),%rax + andq %rsi,%r14 + movq 24(%rcx),%rbx + andq %rsi,%r15 + movq 32(%rcx),%rbp + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%rbx + adcq %r12,%rbp + adcq %r13,%rsi + sbbq %r13,%r13 + + shrq $1,%r14 + movq %r15,%r8 + shrq $1,%r15 + movq %rax,%r9 + shrq $1,%rax + movq %rbx,%r10 + shrq $1,%rbx + movq %rbp,%r11 + shrq $1,%rbp + movq %rsi,%r12 + shrq $1,%rsi + shlq $63,%r8 + shlq $63,%r9 + orq %r14,%r8 + shlq $63,%r10 + orq %r15,%r9 + shlq $63,%r11 + orq %rax,%r10 + shlq $63,%r12 + orq %rbx,%r11 + shlq $63,%r13 + orq %rbp,%r12 + orq %rsi,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _div_by_2_mod_384 +.private_extern _div_by_2_mod_384 + +.p2align 5 +_div_by_2_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq %rdx,%rcx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + call __rshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _lshift_mod_384 +.private_extern _lshift_mod_384 + +.p2align 5 +_lshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +L$oop_lshift_mod_384: + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdi,%rdi + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdi + + movq (%rsp),%rdi + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + decl %edx + jnz L$oop_lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__lshift_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + movq %r8,%r14 + adcq %r11,%r11 + movq %r9,%r15 + adcq %r12,%r12 + movq %r10,%rax + adcq %r13,%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + cmovcq %rbx,%r11 + cmovcq %rbp,%r12 + cmovcq %rsi,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _mul_by_3_mod_384 +.private_extern _mul_by_3_mod_384 + +.p2align 5 +_mul_by_3_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _mul_by_8_mod_384 +.private_extern _mul_by_8_mod_384 + +.p2align 5 +_mul_by_8_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _mul_by_3_mod_384x +.private_extern _mul_by_3_mod_384x + +.p2align 5 +_mul_by_3_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + + movq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq (%rsp),%rsi + leaq 48(%rdi),%rdi + + movq 48(%rsi),%r8 + movq 56(%rsi),%r9 + movq 64(%rsi),%r10 + movq 72(%rsi),%r11 + movq 80(%rsi),%r12 + movq 88(%rsi),%r13 + + call __lshift_mod_384 + + movq $48,%rdx + addq (%rsp),%rdx + call __add_mod_384_a_is_loaded + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _mul_by_8_mod_384x +.private_extern _mul_by_8_mod_384x + +.p2align 5 +_mul_by_8_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq %rdx,%rcx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq (%rsp),%rsi + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + movq %r8,48+0(%rdi) + movq %r9,48+8(%rdi) + movq %r10,48+16(%rdi) + movq %r11,48+24(%rdi) + movq %r12,48+32(%rdi) + movq %r13,48+40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _cneg_mod_384 +.private_extern _cneg_mod_384 + +.p2align 5 +_cneg_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdx +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq %rdx,%r8 + movq 24(%rsi),%r11 + orq %r9,%rdx + movq 32(%rsi),%r12 + orq %r10,%rdx + movq 40(%rsi),%r13 + orq %r11,%rdx + movq $-1,%rsi + orq %r12,%rdx + orq %r13,%rdx + + movq 0(%rcx),%r14 + cmovnzq %rsi,%rdx + movq 8(%rcx),%r15 + movq 16(%rcx),%rax + andq %rdx,%r14 + movq 24(%rcx),%rbx + andq %rdx,%r15 + movq 32(%rcx),%rbp + andq %rdx,%rax + movq 40(%rcx),%rsi + andq %rdx,%rbx + movq 0(%rsp),%rcx + andq %rdx,%rbp + andq %rdx,%rsi + + subq %r8,%r14 + sbbq %r9,%r15 + sbbq %r10,%rax + sbbq %r11,%rbx + sbbq %r12,%rbp + sbbq %r13,%rsi + + orq %rcx,%rcx + + cmovzq %r8,%r14 + cmovzq %r9,%r15 + cmovzq %r10,%rax + movq %r14,0(%rdi) + cmovzq %r11,%rbx + movq %r15,8(%rdi) + cmovzq %r12,%rbp + movq %rax,16(%rdi) + cmovzq %r13,%rsi + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rsi,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.globl _sub_mod_384 +.private_extern _sub_mod_384 + +.p2align 5 +_sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sub_mod_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sub_mod_384x +.private_extern _sub_mod_384x + +.p2align 5 +_sub_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 24 + + + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + leaq 48(%rsi),%rsi + leaq 48(%rdx),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384 + + movq 0(%rsp),%rsi + movq 8(%rsp),%rdx + leaq -48(%rdi),%rdi + call __sub_mod_384 + + movq 24+0(%rsp),%r15 +.cfi_restore %r15 + movq 24+8(%rsp),%r14 +.cfi_restore %r14 + movq 24+16(%rsp),%r13 +.cfi_restore %r13 + movq 24+24(%rsp),%r12 +.cfi_restore %r12 + movq 24+32(%rsp),%rbx +.cfi_restore %rbx + movq 24+40(%rsp),%rbp +.cfi_restore %rbp + leaq 24+48(%rsp),%rsp +.cfi_adjust_cfa_offset -24-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mul_by_1_plus_i_mod_384x +.private_extern _mul_by_1_plus_i_mod_384x + +.p2align 5 +_mul_by_1_plus_i_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $56,%rsp +.cfi_adjust_cfa_offset 56 + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rbx + adcq 72(%rsi),%r11 + movq %r12,%rcx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + movq %rdi,48(%rsp) + sbbq %rdi,%rdi + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rbx + sbbq 80(%rsi),%rcx + sbbq 88(%rsi),%rbp + sbbq %rsi,%rsi + + movq %r8,0(%rsp) + movq 0(%rdx),%r8 + movq %r9,8(%rsp) + movq 8(%rdx),%r9 + movq %r10,16(%rsp) + movq 16(%rdx),%r10 + movq %r11,24(%rsp) + movq 24(%rdx),%r11 + movq %r12,32(%rsp) + andq %rsi,%r8 + movq 32(%rdx),%r12 + movq %r13,40(%rsp) + andq %rsi,%r9 + movq 40(%rdx),%r13 + andq %rsi,%r10 + andq %rsi,%r11 + andq %rsi,%r12 + andq %rsi,%r13 + movq 48(%rsp),%rsi + + addq %r8,%r14 + movq 0(%rsp),%r8 + adcq %r9,%r15 + movq 8(%rsp),%r9 + adcq %r10,%rax + movq 16(%rsp),%r10 + adcq %r11,%rbx + movq 24(%rsp),%r11 + adcq %r12,%rcx + movq 32(%rsp),%r12 + adcq %r13,%rbp + movq 40(%rsp),%r13 + + movq %r14,0(%rsi) + movq %r8,%r14 + movq %r15,8(%rsi) + movq %rax,16(%rsi) + movq %r9,%r15 + movq %rbx,24(%rsi) + movq %rcx,32(%rsi) + movq %r10,%rax + movq %rbp,40(%rsi) + + subq 0(%rdx),%r8 + movq %r11,%rbx + sbbq 8(%rdx),%r9 + sbbq 16(%rdx),%r10 + movq %r12,%rcx + sbbq 24(%rdx),%r11 + sbbq 32(%rdx),%r12 + movq %r13,%rbp + sbbq 40(%rdx),%r13 + sbbq $0,%rdi + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,48(%rsi) + cmovcq %rbx,%r11 + movq %r9,56(%rsi) + cmovcq %rcx,%r12 + movq %r10,64(%rsi) + cmovcq %rbp,%r13 + movq %r11,72(%rsi) + movq %r12,80(%rsi) + movq %r13,88(%rsi) + + movq 56+0(%rsp),%r15 +.cfi_restore %r15 + movq 56+8(%rsp),%r14 +.cfi_restore %r14 + movq 56+16(%rsp),%r13 +.cfi_restore %r13 + movq 56+24(%rsp),%r12 +.cfi_restore %r12 + movq 56+32(%rsp),%rbx +.cfi_restore %rbx + movq 56+40(%rsp),%rbp +.cfi_restore %rbp + leaq 56+48(%rsp),%rsp +.cfi_adjust_cfa_offset -56-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sgn0_pty_mod_384 +.private_extern _sgn0_pty_mod_384 + +.p2align 5 +_sgn0_pty_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%rcx + movq 40(%rdi),%rdx + + xorq %rax,%rax + movq %r8,%rdi + addq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + notq %rax + andq $1,%rdi + andq $2,%rax + orq %rdi,%rax + + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sgn0_pty_mod_384x +.private_extern _sgn0_pty_mod_384x + +.p2align 5 +_sgn0_pty_mod_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq 48(%rdi),%r8 + movq 56(%rdi),%r9 + movq 64(%rdi),%r10 + movq 72(%rdi),%r11 + movq 80(%rdi),%rcx + movq 88(%rdi),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + leaq 0(%rdi),%rax + xorq %rdi,%rdi + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rdi + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rdi + + movq %r8,0(%rsp) + notq %rdi + andq $1,%rbp + andq $2,%rdi + orq %rbp,%rdi + + movq 0(%rax),%r8 + movq 8(%rax),%r9 + movq 16(%rax),%r10 + movq 24(%rax),%r11 + movq 32(%rax),%rcx + movq 40(%rax),%rdx + + movq %r8,%rbx + orq %r9,%r8 + orq %r10,%r8 + orq %r11,%r8 + orq %rcx,%r8 + orq %rdx,%r8 + + xorq %rax,%rax + movq %rbx,%rbp + addq %rbx,%rbx + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %rcx,%rcx + adcq %rdx,%rdx + adcq $0,%rax + + subq 0(%rsi),%rbx + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + sbbq 24(%rsi),%r11 + sbbq 32(%rsi),%rcx + sbbq 40(%rsi),%rdx + sbbq $0,%rax + + movq 0(%rsp),%rbx + + notq %rax + + testq %r8,%r8 + cmovzq %rdi,%rbp + + testq %rbx,%rbx + cmovnzq %rdi,%rax + + andq $1,%rbp + andq $2,%rax + orq %rbp,%rax + + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_32 +.private_extern _vec_select_32 + +.p2align 5 +_vec_select_32: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 16(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 16(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 16(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-16(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-16(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-16(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,16-16(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_48 +.private_extern _vec_select_48 + +.p2align 5 +_vec_select_48: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 24(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 24(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 24(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-24(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-24(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-24(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-24(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-24(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-24(%rdi) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,32-24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_96 +.private_extern _vec_select_96 + +.p2align 5 +_vec_select_96: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 48(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 48(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 48(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-48(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-48(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-48(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-48(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-48(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-48(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-48(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-48(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-48(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-48(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-48(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,80-48(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_192 +.private_extern _vec_select_192 + +.p2align 5 +_vec_select_192: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 96(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 96(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 96(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-96(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-96(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-96(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-96(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-96(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-96(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-96(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-96(%rdi) + pand %xmm4,%xmm0 + movdqu 128+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-96(%rdi) + pand %xmm4,%xmm2 + movdqu 144+16-96(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-96(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-96(%rdi) + pand %xmm4,%xmm0 + movdqu 160+16-96(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-96(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-96(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,176-96(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_144 +.private_extern _vec_select_144 + +.p2align 5 +_vec_select_144: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 72(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 72(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 72(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-72(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-72(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-72(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-72(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-72(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-72(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-72(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-72(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-72(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-72(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-72(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-72(%rdi) + pand %xmm4,%xmm0 + pand %xmm5,%xmm1 + por %xmm1,%xmm0 + movdqu %xmm0,128-72(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_select_288 +.private_extern _vec_select_288 + +.p2align 5 +_vec_select_288: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movd %ecx,%xmm5 + pxor %xmm4,%xmm4 + pshufd $0,%xmm5,%xmm5 + movdqu (%rsi),%xmm0 + leaq 144(%rsi),%rsi + pcmpeqd %xmm4,%xmm5 + movdqu (%rdx),%xmm1 + leaq 144(%rdx),%rdx + pcmpeqd %xmm5,%xmm4 + leaq 144(%rdi),%rdi + pand %xmm4,%xmm0 + movdqu 0+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 0+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,0-144(%rdi) + pand %xmm4,%xmm2 + movdqu 16+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 16+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,16-144(%rdi) + pand %xmm4,%xmm0 + movdqu 32+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 32+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,32-144(%rdi) + pand %xmm4,%xmm2 + movdqu 48+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 48+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,48-144(%rdi) + pand %xmm4,%xmm0 + movdqu 64+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 64+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,64-144(%rdi) + pand %xmm4,%xmm2 + movdqu 80+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 80+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,80-144(%rdi) + pand %xmm4,%xmm0 + movdqu 96+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 96+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,96-144(%rdi) + pand %xmm4,%xmm2 + movdqu 112+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 112+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,112-144(%rdi) + pand %xmm4,%xmm0 + movdqu 128+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 128+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,128-144(%rdi) + pand %xmm4,%xmm2 + movdqu 144+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 144+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,144-144(%rdi) + pand %xmm4,%xmm0 + movdqu 160+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 160+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,160-144(%rdi) + pand %xmm4,%xmm2 + movdqu 176+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 176+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,176-144(%rdi) + pand %xmm4,%xmm0 + movdqu 192+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 192+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,192-144(%rdi) + pand %xmm4,%xmm2 + movdqu 208+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 208+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,208-144(%rdi) + pand %xmm4,%xmm0 + movdqu 224+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 224+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,224-144(%rdi) + pand %xmm4,%xmm2 + movdqu 240+16-144(%rsi),%xmm0 + pand %xmm5,%xmm3 + movdqu 240+16-144(%rdx),%xmm1 + por %xmm3,%xmm2 + movdqu %xmm2,240-144(%rdi) + pand %xmm4,%xmm0 + movdqu 256+16-144(%rsi),%xmm2 + pand %xmm5,%xmm1 + movdqu 256+16-144(%rdx),%xmm3 + por %xmm1,%xmm0 + movdqu %xmm0,256-144(%rdi) + pand %xmm4,%xmm2 + pand %xmm5,%xmm3 + por %xmm3,%xmm2 + movdqu %xmm2,272-144(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_prefetch +.private_extern _vec_prefetch + +.p2align 5 +_vec_prefetch: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + leaq -1(%rdi,%rsi,1),%rsi + movq $64,%rax + xorq %r8,%r8 + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + cmovaq %r8,%rax + prefetchnta (%rdi) + leaq (%rdi,%rax,1),%rdi + cmpq %rsi,%rdi + cmovaq %rsi,%rdi + prefetchnta (%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_is_zero_16x +.private_extern _vec_is_zero_16x + +.p2align 5 +_vec_is_zero_16x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%esi + movdqu (%rdi),%xmm0 + leaq 16(%rdi),%rdi + +L$oop_is_zero: + decl %esi + jz L$oop_is_zero_done + movdqu (%rdi),%xmm1 + leaq 16(%rdi),%rdi + por %xmm1,%xmm0 + jmp L$oop_is_zero + +L$oop_is_zero_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %esi + testq %rax,%rax + cmovnzl %esi,%eax + xorl $1,%eax + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _vec_is_equal_16x +.private_extern _vec_is_equal_16x + +.p2align 5 +_vec_is_equal_16x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + shrl $4,%edx + movdqu (%rdi),%xmm0 + movdqu (%rsi),%xmm1 + subq %rdi,%rsi + leaq 16(%rdi),%rdi + pxor %xmm1,%xmm0 + +L$oop_is_equal: + decl %edx + jz L$oop_is_equal_done + movdqu (%rdi),%xmm1 + movdqu (%rdi,%rsi,1),%xmm2 + leaq 16(%rdi),%rdi + pxor %xmm2,%xmm1 + por %xmm1,%xmm0 + jmp L$oop_is_equal + +L$oop_is_equal_done: + pshufd $0x4e,%xmm0,%xmm1 + por %xmm1,%xmm0 +.byte 102,72,15,126,192 + incl %edx + testq %rax,%rax + cmovnzl %edx,%eax + xorl $1,%eax + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/add_mod_384x384-x86_64.s b/crypto/blst_src/build/mach-o/add_mod_384x384-x86_64.s new file mode 100644 index 00000000000..2dc58f81608 --- /dev/null +++ b/crypto/blst_src/build/mach-o/add_mod_384x384-x86_64.s @@ -0,0 +1,244 @@ +.text + + +.p2align 5 +__add_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + addq 0(%rdx),%r8 + movq 56(%rsi),%r15 + adcq 8(%rdx),%r9 + movq 64(%rsi),%rax + adcq 16(%rdx),%r10 + movq 72(%rsi),%rbx + adcq 24(%rdx),%r11 + movq 80(%rsi),%rbp + adcq 32(%rdx),%r12 + movq 88(%rsi),%rsi + adcq 40(%rdx),%r13 + movq %r8,0(%rdi) + adcq 48(%rdx),%r14 + movq %r9,8(%rdi) + adcq 56(%rdx),%r15 + movq %r10,16(%rdi) + adcq 64(%rdx),%rax + movq %r12,32(%rdi) + movq %r14,%r8 + adcq 72(%rdx),%rbx + movq %r11,24(%rdi) + movq %r15,%r9 + adcq 80(%rdx),%rbp + movq %r13,40(%rdi) + movq %rax,%r10 + adcq 88(%rdx),%rsi + movq %rbx,%r11 + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %rbp,%r12 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%rbx + sbbq 32(%rcx),%rbp + movq %rsi,%r13 + sbbq 40(%rcx),%rsi + sbbq $0,%rdx + + cmovcq %r8,%r14 + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %r14,48(%rdi) + cmovcq %r11,%rbx + movq %r15,56(%rdi) + cmovcq %r12,%rbp + movq %rax,64(%rdi) + cmovcq %r13,%rsi + movq %rbx,72(%rdi) + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _add_mod_384x384 +.private_extern _add_mod_384x384 + +.p2align 5 +_add_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __add_mod_384x384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sub_mod_384x384 +.private_extern _sub_mod_384x384 + +.p2align 5 +_sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sub_mod_384x384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S b/crypto/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S new file mode 100644 index 00000000000..f3a2c3b5f11 --- /dev/null +++ b/crypto/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S @@ -0,0 +1,784 @@ +.text + +.globl _ct_inverse_mod_256 + +.align 5 +_ct_inverse_mod_256: +.long 3573752639 + stp x29, x30, [sp,#-80]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + sub sp, sp, #1040 + + ldp x4, x5, [x1,#8*0] + ldp x6, x7, [x1,#8*2] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + str x0, [sp] + + ldp x8, x9, [x2,#8*0] + ldp x10, x11, [x2,#8*2] + + stp x4, x5, [x1,#8*0] // copy input to |a| + stp x6, x7, [x1,#8*2] + stp x8, x9, [x1,#8*4] // copy modulus to |b| + stp x10, x11, [x1,#8*6] + + ////////////////////////////////////////// first iteration + bl Lab_approximation_31_256_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + str x12,[x0,#8*8] // initialize |u| with |f0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to dst |b| + bl __smul_256_n_shift_by_31 + str x12, [x0,#8*9] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + ldr x8, [x1,#8*8] // |u| + ldr x9, [x1,#8*13] // |v| + madd x4, x16, x8, xzr // |u|*|f0| + madd x4, x17, x9, x4 // |v|*|g0| + str x4, [x0,#8*4] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*5] + stp x5, x5, [x0,#8*7] + + madd x4, x12, x8, xzr // |u|*|f1| + madd x4, x13, x9, x4 // |v|*|g1| + str x4, [x0,#8*9] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*10] + stp x5, x5, [x0,#8*12] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + ////////////////////////////////////////// two[!] last iterations + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #47 // 31 + 512 % 31 + //bl __ab_approximation_62_256 // |a| and |b| are exact, + ldr x7, [x1,#8*0] // just load + ldr x11, [x1,#8*4] + bl __inner_loop_62_256 + + mov x16, x14 + mov x17, x15 + ldr x0, [sp] // original out_ptr + bl __smul_256x63 + bl __smul_512x63_tail + ldr x30, [x29,#8] + + smulh x20, x7, x17 // figure out top-most limb + ldp x8, x9, [x3,#8*0] + adc x23, x23, x25 + ldp x10, x11, [x3,#8*2] + + add x20, x20, x23 // x20 is 1, 0 or -1 + asr x19, x20, #63 // sign as mask + + and x23, x8, x19 // add mod<<256 conditionally + and x24, x9, x19 + adds x4, x4, x23 + and x25, x10, x19 + adcs x5, x5, x24 + and x26, x11, x19 + adcs x6, x6, x25 + adcs x7, x22, x26 + adc x20, x20, xzr // x20 is 1, 0 or -1 + + neg x19, x20 + orr x20, x20, x19 // excess bit or sign as mask + asr x19, x19, #63 // excess bit as mask + + and x8, x8, x20 // mask |mod| + and x9, x9, x20 + and x10, x10, x20 + and x11, x11, x20 + + eor x8, x8, x19 // conditionally negate |mod| + eor x9, x9, x19 + adds x8, x8, x19, lsr#63 + eor x10, x10, x19 + adcs x9, x9, xzr + eor x11, x11, x19 + adcs x10, x10, xzr + adc x11, x11, xzr + + adds x4, x4, x8 // final adjustment for |mod|<<256 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*4] + adc x7, x7, x11 + stp x6, x7, [x0,#8*6] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldr x29, [sp],#80 +.long 3573752767 + ret + + +//////////////////////////////////////////////////////////////////////// + +.align 5 +__smul_256x63: + ldp x4, x5, [x1,#8*0+64] // load |u| (or |v|) + asr x14, x16, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x6, x7, [x1,#8*2+64] + eor x16, x16, x14 // conditionally negate |f_| (or |g_|) + ldr x22, [x1,#8*4+64] + + eor x4, x4, x14 // conditionally negate |u| (or |v|) + sub x16, x16, x14 + eor x5, x5, x14 + adds x4, x4, x14, lsr#63 + eor x6, x6, x14 + adcs x5, x5, xzr + eor x7, x7, x14 + adcs x6, x6, xzr + eor x22, x22, x14 + umulh x19, x4, x16 + adcs x7, x7, xzr + umulh x20, x5, x16 + adcs x22, x22, xzr + umulh x21, x6, x16 + mul x4, x4, x16 + cmp x16, #0 + mul x5, x5, x16 + csel x22, x22, xzr, ne + mul x6, x6, x16 + adds x5, x5, x19 + mul x24, x7, x16 + adcs x6, x6, x20 + adcs x24, x24, x21 + adc x26, xzr, xzr + ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x10, x11, [x1,#8*2+104] + eor x17, x17, x14 // conditionally negate |f_| (or |g_|) + ldr x23, [x1,#8*4+104] + + eor x8, x8, x14 // conditionally negate |u| (or |v|) + sub x17, x17, x14 + eor x9, x9, x14 + adds x8, x8, x14, lsr#63 + eor x10, x10, x14 + adcs x9, x9, xzr + eor x11, x11, x14 + adcs x10, x10, xzr + eor x23, x23, x14 + umulh x19, x8, x17 + adcs x11, x11, xzr + umulh x20, x9, x17 + adcs x23, x23, xzr + umulh x21, x10, x17 + adc x15, xzr, xzr // used in __smul_512x63_tail + mul x8, x8, x17 + cmp x17, #0 + mul x9, x9, x17 + csel x23, x23, xzr, ne + mul x10, x10, x17 + adds x9, x9, x19 + mul x25, x11, x17 + adcs x10, x10, x20 + adcs x25, x25, x21 + adc x26, x26, xzr + + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*0] + adcs x24, x24, x25 + stp x6, x24, [x0,#8*2] + + ret + + + +.align 5 +__smul_512x63_tail: + umulh x24, x7, x16 + ldp x5, x6, [x1,#8*18] // load rest of |v| + adc x26, x26, xzr + ldr x7, [x1,#8*20] + and x22, x22, x16 + + umulh x11, x11, x17 // resume |v|*|g1| chain + + sub x24, x24, x22 // tie up |u|*|f1| chain + asr x25, x24, #63 + + eor x5, x5, x14 // conditionally negate rest of |v| + eor x6, x6, x14 + adds x5, x5, x15 + eor x7, x7, x14 + adcs x6, x6, xzr + umulh x19, x23, x17 + adc x7, x7, xzr + umulh x20, x5, x17 + add x11, x11, x26 + umulh x21, x6, x17 + + mul x4, x23, x17 + mul x5, x5, x17 + adds x4, x4, x11 + mul x6, x6, x17 + adcs x5, x5, x19 + mul x22, x7, x17 + adcs x6, x6, x20 + adcs x22, x22, x21 + adc x23, xzr, xzr // used in the final step + + adds x4, x4, x24 + adcs x5, x5, x25 + adcs x6, x6, x25 + stp x4, x5, [x0,#8*4] + adcs x22, x22, x25 // carry is used in the final step + stp x6, x22, [x0,#8*6] + + ret + + + +.align 5 +__smul_256_n_shift_by_31: + ldp x4, x5, [x1,#8*0+0] // load |a| (or |b|) + asr x24, x12, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x6, x7, [x1,#8*2+0] + eor x25, x12, x24 // conditionally negate |f0| (or |g0|) + + eor x4, x4, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x5, x5, x24 + adds x4, x4, x24, lsr#63 + eor x6, x6, x24 + adcs x5, x5, xzr + eor x7, x7, x24 + umulh x19, x4, x25 + adcs x6, x6, xzr + umulh x20, x5, x25 + adc x7, x7, xzr + umulh x21, x6, x25 + and x24, x24, x25 + umulh x22, x7, x25 + neg x24, x24 + + mul x4, x4, x25 + mul x5, x5, x25 + mul x6, x6, x25 + adds x5, x5, x19 + mul x7, x7, x25 + adcs x6, x6, x20 + adcs x7, x7, x21 + adc x22, x22, x24 + ldp x8, x9, [x1,#8*0+32] // load |a| (or |b|) + asr x24, x13, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x10, x11, [x1,#8*2+32] + eor x25, x13, x24 // conditionally negate |f0| (or |g0|) + + eor x8, x8, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x9, x9, x24 + adds x8, x8, x24, lsr#63 + eor x10, x10, x24 + adcs x9, x9, xzr + eor x11, x11, x24 + umulh x19, x8, x25 + adcs x10, x10, xzr + umulh x20, x9, x25 + adc x11, x11, xzr + umulh x21, x10, x25 + and x24, x24, x25 + umulh x23, x11, x25 + neg x24, x24 + + mul x8, x8, x25 + mul x9, x9, x25 + mul x10, x10, x25 + adds x9, x9, x19 + mul x11, x11, x25 + adcs x10, x10, x20 + adcs x11, x11, x21 + adc x23, x23, x24 + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + adcs x7, x7, x11 + adc x8, x22, x23 + + extr x4, x5, x4, #31 + extr x5, x6, x5, #31 + extr x6, x7, x6, #31 + asr x23, x8, #63 // result's sign as mask + extr x7, x8, x7, #31 + + eor x4, x4, x23 // ensure the result is positive + eor x5, x5, x23 + adds x4, x4, x23, lsr#63 + eor x6, x6, x23 + adcs x5, x5, xzr + eor x7, x7, x23 + adcs x6, x6, xzr + stp x4, x5, [x0,#8*0] + adc x7, x7, xzr + stp x6, x7, [x0,#8*2] + + eor x12, x12, x23 // adjust |f/g| accordingly + eor x13, x13, x23 + sub x12, x12, x23 + sub x13, x13, x23 + + ret + + +.align 4 +__ab_approximation_31_256: + ldp x6, x7, [x1,#8*2] + ldp x10, x11, [x1,#8*6] + ldp x4, x5, [x1,#8*0] + ldp x8, x9, [x1,#8*4] + +Lab_approximation_31_256_loaded: + orr x19, x7, x11 // check top-most limbs, ... + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x5, ne + orr x19, x7, x11 // and ones before top-most, ... + csel x10, x10, x9, ne + + cmp x19, #0 + csel x7, x7, x6, ne + csel x11, x11, x10, ne + csel x6, x6, x4, ne + orr x19, x7, x11 // and one more, ... + csel x10, x10, x8, ne + + clz x19, x19 + cmp x19, #64 + csel x19, x19, xzr, ne + csel x7, x7, x6, ne + csel x11, x11, x10, ne + neg x20, x19 + + lslv x7, x7, x19 // align high limbs to the left + lslv x11, x11, x19 + lsrv x6, x6, x20 + lsrv x10, x10, x20 + and x6, x6, x20, asr#6 + and x10, x10, x20, asr#6 + orr x7, x7, x6 + orr x11, x11, x10 + + bfxil x7, x4, #0, #31 + bfxil x11, x8, #0, #31 + + b __inner_loop_31_256 + ret + + + +.align 4 +__inner_loop_31_256: + mov x2, #31 + mov x13, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x15, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x23,#0x7FFFFFFF7FFFFFFF + +Loop_31_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x15 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x15, x15, x13, hs // exchange |fg0| and |fg1| + csel x13, x13, x19, hs + lsr x7, x7, #1 + and x19, x15, x22 + and x20, x23, x22 + sub x13, x13, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x15, x15, x15 // |f1|<<=1 + add x13, x13, x20 + sub x15, x15, x23 + cbnz x2, Loop_31_256 + + mov x23, #0x7FFFFFFF + ubfx x12, x13, #0, #32 + ubfx x13, x13, #32, #32 + ubfx x14, x15, #0, #32 + ubfx x15, x15, #32, #32 + sub x12, x12, x23 // remove bias + sub x13, x13, x23 + sub x14, x14, x23 + sub x15, x15, x23 + + ret + + + +.align 4 +__inner_loop_62_256: + mov x12, #1 // |f0|=1 + mov x13, #0 // |g0|=0 + mov x14, #0 // |f1|=0 + mov x15, #1 // |g1|=1 + +Loop_62_256: + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x12 + csel x11, x11, x7, hs // |b_| = |a_| + csel x7, x21, x20, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + mov x20, x13 + csel x12, x12, x14, hs // exchange |f0| and |f1| + csel x14, x14, x19, hs + csel x13, x13, x15, hs // exchange |g0| and |g1| + csel x15, x15, x20, hs + lsr x7, x7, #1 + and x19, x14, x22 + and x20, x15, x22 + add x14, x14, x14 // |f1|<<=1 + add x15, x15, x15 // |g1|<<=1 + sub x12, x12, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x13, x13, x20 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, Loop_62_256 + + ret + diff --git a/crypto/blst_src/build/mach-o/ct_inverse_mod_256-x86_64.s b/crypto/blst_src/build/mach-o/ct_inverse_mod_256-x86_64.s new file mode 100644 index 00000000000..b6441da6e1f --- /dev/null +++ b/crypto/blst_src/build/mach-o/ct_inverse_mod_256-x86_64.s @@ -0,0 +1,1177 @@ +.text + +.globl _ct_inverse_mod_256 + +.p2align 5 +_ct_inverse_mod_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1072,%rsp +.cfi_adjust_cfa_offset 1072 + + + leaq 48+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + + movq 0(%rdx),%r12 + movq 8(%rdx),%r13 + movq 16(%rdx),%r14 + movq 24(%rdx),%r15 + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + + movq %r12,32(%rax) + movq %r13,40(%rax) + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rax,%rsi + + + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,64(%rdi) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + movq %rdx,72(%rdi) + + + xorq $256,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + + + + movq 64(%rsi),%r8 + movq 104(%rsi),%r12 + movq %r8,%r9 + imulq 0(%rsp),%r8 + movq %r12,%r13 + imulq 8(%rsp),%r12 + addq %r12,%r8 + movq %r8,32(%rdi) + sarq $63,%r8 + movq %r8,40(%rdi) + movq %r8,48(%rdi) + movq %r8,56(%rdi) + movq %r8,64(%rdi) + leaq 64(%rsi),%rsi + + imulq %rdx,%r9 + imulq %rcx,%r13 + addq %r13,%r9 + movq %r9,72(%rdi) + sarq $63,%r9 + movq %r9,80(%rdi) + movq %r9,88(%rdi) + movq %r9,96(%rdi) + movq %r9,104(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_256x63 + sarq $63,%rbp + movq %rbp,40(%rdi) + movq %rbp,48(%rdi) + movq %rbp,56(%rdi) + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + xorq $256+64,%rsi + movl $31,%edx + call __ab_approximation_31_256 + + + movq %r12,16(%rsp) + movq %r13,24(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,0(%rsp) + movq %rcx,8(%rsp) + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 32(%rdi),%rdi + call __smulq_256_n_shift_by_31 + movq %rdx,16(%rsp) + movq %rcx,24(%rsp) + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq 64(%rsi),%rsi + leaq 32(%rdi),%rdi + call __smulq_256x63 + + movq 16(%rsp),%rdx + movq 24(%rsp),%rcx + leaq 40(%rdi),%rdi + call __smulq_512x63 + + xorq $256+64,%rsi + movl $47,%edx + + movq 0(%rsi),%r8 + + movq 32(%rsi),%r10 + + call __inner_loop_62_256 + + + + + + + + leaq 64(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_512x63 + adcq %rbp,%rdx + + movq 40(%rsp),%rsi + movq %rdx,%rax + sarq $63,%rdx + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + adcq $0,%rax + + movq %rax,%rdx + negq %rax + orq %rax,%rdx + sarq $63,%rax + + movq %rdx,%r8 + movq %rdx,%r9 + andq 0(%rsi),%r8 + movq %rdx,%r10 + andq 8(%rsi),%r9 + andq 16(%rsi),%r10 + andq 24(%rsi),%rdx + + xorq %rax,%r8 + xorq %rcx,%rcx + xorq %rax,%r9 + subq %rax,%rcx + xorq %rax,%r10 + xorq %rax,%rdx + addq %rcx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%rdx + + addq %r8,%r12 + adcq %r9,%r13 + adcq %r10,%r14 + adcq %rdx,%r15 + + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + leaq 1072(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1072-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulq_512x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %r9,8(%rdi) + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %r10,16(%rdi) + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %r11,24(%rdi) + + movq 40(%rsi),%r8 + movq 48(%rsi),%r9 + movq 56(%rsi),%r10 + movq 64(%rsi),%r11 + movq 72(%rsi),%r12 + movq 80(%rsi),%r13 + movq 88(%rsi),%r14 + movq 96(%rsi),%r15 + + movq %rcx,%rdx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rcx + addq %rax,%rcx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rcx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rcx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rcx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rcx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rcx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rcx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rcx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + imulq %rcx + addq %rax,%r15 + adcq $0,%rdx + + movq %rbp,%rbx + sarq $63,%rbp + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq %rbx,%r12 + adcq %rbp,%r13 + adcq %rbp,%r14 + adcq %rbp,%r15 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__smulq_256x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%rbp + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%rbp + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%rbp + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + andq %rbx,%rbp + negq %rbp + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq %rcx,%rdx + movq 40+0(%rsi),%r12 + movq 40+8(%rsi),%r13 + movq 40+16(%rsi),%r14 + movq 40+24(%rsi),%r15 + movq 40+32(%rsi),%rcx + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rcx + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rcx + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + andq %rbx,%rcx + negq %rcx + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %rbp,32(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulq_256_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,0(%rdi) + movq %rcx,8(%rdi) + movq %rdx,%rbp + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + + movq %rbp,%rbx + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rbx + addq %rax,%rbx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + andq %rbx,%rbp + negq %rbp + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + adcq %rdx,%rbp + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r14 + movq 32+24(%rsi),%r15 + + movq %rcx,%rbx + sarq $63,%rcx + xorq %rax,%rax + subq %rcx,%rax + + xorq %rcx,%rbx + addq %rax,%rbx + + xorq %rcx,%r12 + xorq %rcx,%r13 + xorq %rcx,%r14 + xorq %rcx,%r15 + addq %r12,%rax + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + + mulq %rbx + movq %rax,%r12 + movq %r13,%rax + andq %rbx,%rcx + negq %rcx + movq %rdx,%r13 + mulq %rbx + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rbx + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rbx + addq %rax,%r15 + adcq %rdx,%rcx + addq %r12,%r8 + adcq %r13,%r9 + adcq %r14,%r10 + adcq %r15,%r11 + adcq %rcx,%rbp + + movq 0(%rdi),%rdx + movq 8(%rdi),%rcx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%rbp,%r11 + + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + xorq %rbp,%rdx + xorq %rbp,%rcx + addq %rax,%rdx + addq %rax,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__ab_approximation_31_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 24(%rsi),%r9 + movq 56(%rsi),%r11 + movq 16(%rsi),%rbx + movq 48(%rsi),%rbp + movq 8(%rsi),%r8 + movq 40(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 32(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + notq %rax + andq %rax,%r9 + andq %rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31_256 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__inner_loop_31_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +L$oop_31_256: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edx + jnz L$oop_31_256 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__inner_loop_62_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movl %edx,%r15d + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq %rdx,%r13 + movq %rdx,%r14 + +L$oop_62_256: + xorq %rax,%rax + testq %r14,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq %r14,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%r15d + jnz L$oop_62_256 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S b/crypto/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S new file mode 100644 index 00000000000..c7d9ba8488e --- /dev/null +++ b/crypto/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S @@ -0,0 +1,717 @@ +.text + +.globl _ct_inverse_mod_383 + +.align 5 +_ct_inverse_mod_383: +.long 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #1040 + + ldp x22, x4, [x1,#8*0] + ldp x5, x6, [x1,#8*2] + ldp x7, x8, [x1,#8*4] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + stp x0, x3, [sp] + + ldp x9, x10, [x2,#8*0] + ldp x11, x12, [x2,#8*2] + ldp x13, x14, [x2,#8*4] + + stp x22, x4, [x1,#8*0] // copy input to |a| + stp x5, x6, [x1,#8*2] + stp x7, x8, [x1,#8*4] + stp x9, x10, [x1,#8*6] // copy modulus to |b| + stp x11, x12, [x1,#8*8] + stp x13, x14, [x1,#8*10] + + ////////////////////////////////////////// first iteration + mov x2, #62 + bl Lab_approximation_62_loaded + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + str x15,[x0,#8*12] // initialize |u| with |f0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to dst |b| + bl __smul_383_n_shift_by_62 + str x15, [x0,#8*12] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + ldr x7, [x1,#8*12] // |u| + ldr x8, [x1,#8*18] // |v| + mul x3, x20, x7 // |u|*|f0| + smulh x4, x20, x7 + mul x5, x21, x8 // |v|*|g0| + smulh x6, x21, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*6] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*8] + stp x5, x5, [x0,#8*10] + + mul x3, x15, x7 // |u|*|f1| + smulh x4, x15, x7 + mul x5, x16, x8 // |v|*|g1| + smulh x6, x16, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*12] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*14] + stp x5, x5, [x0,#8*16] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + asr x27, x27, #63 // sign extension + stp x27, x27, [x0,#8*6] + stp x27, x27, [x0,#8*8] + stp x27, x27, [x0,#8*10] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + ////////////////////////////////////////// iteration before last + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldp x3, x8, [x1,#8*0] // just load + ldp x9, x14, [x1,#8*6] + bl __inner_loop_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + str x3, [x0,#8*0] + str x9, [x0,#8*6] + + mov x20, x15 // exact |f0| + mov x21, x16 // exact |g0| + mov x15, x17 + mov x16, x19 + add x0, x0, #8*12 // pointer to dst |u| + bl __smul_383x63 + + mov x20, x15 // exact |f1| + mov x21, x16 // exact |g1| + add x0, x0, #8*6 // pointer to dst |v| + bl __smul_383x63 + bl __smul_767x63_tail + + ////////////////////////////////////////// last iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #22 // 766 % 62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldr x3, [x1,#8*0] // just load + eor x8, x8, x8 + ldr x9, [x1,#8*6] + eor x14, x14, x14 + bl __inner_loop_62 + + mov x20, x17 + mov x21, x19 + ldp x0, x15, [sp] // original out_ptr and n_ptr + bl __smul_383x63 + bl __smul_767x63_tail + ldr x30, [x29,#8] + + asr x22, x8, #63 // sign as mask + ldp x9, x10, [x15,#8*0] + ldp x11, x12, [x15,#8*2] + ldp x13, x14, [x15,#8*4] + + and x9, x9, x22 // add mod<<384 conditionally + and x10, x10, x22 + adds x3, x3, x9 + and x11, x11, x22 + adcs x4, x4, x10 + and x12, x12, x22 + adcs x5, x5, x11 + and x13, x13, x22 + adcs x6, x6, x12 + and x14, x14, x22 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*8] + adc x8, x8, x14 + stp x7, x8, [x0,#8*10] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 +.long 3573752767 + ret + + +//////////////////////////////////////////////////////////////////////// +// see corresponding commentary in ctx_inverse_mod_384-x86_64... + +.align 5 +__smul_383x63: + ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) + asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x5, x6, [x1,#8*2+96] + eor x20, x20, x17 // conditionally negate |f_| (or |g_|) + ldp x7, x8, [x1,#8*4+96] + + eor x3, x3, x17 // conditionally negate |u| (or |v|) + sub x20, x20, x17 + eor x4, x4, x17 + adds x3, x3, x17, lsr#63 + eor x5, x5, x17 + adcs x4, x4, xzr + eor x6, x6, x17 + adcs x5, x5, xzr + eor x7, x7, x17 + adcs x6, x6, xzr + umulh x22, x3, x20 + eor x8, x8, x17 + umulh x23, x4, x20 + adcs x7, x7, xzr + umulh x24, x5, x20 + adcs x8, x8, xzr + umulh x25, x6, x20 + umulh x26, x7, x20 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x22 + mul x6, x6, x20 + adcs x5, x5, x23 + mul x7, x7, x20 + adcs x6, x6, x24 + mul x27,x8, x20 + adcs x7, x7, x25 + adcs x27,x27,x26 + adc x2, xzr, xzr + ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x11, x12, [x1,#8*2+144] + eor x21, x21, x17 // conditionally negate |f_| (or |g_|) + ldp x13, x14, [x1,#8*4+144] + + eor x9, x9, x17 // conditionally negate |u| (or |v|) + sub x21, x21, x17 + eor x10, x10, x17 + adds x9, x9, x17, lsr#63 + eor x11, x11, x17 + adcs x10, x10, xzr + eor x12, x12, x17 + adcs x11, x11, xzr + eor x13, x13, x17 + adcs x12, x12, xzr + umulh x22, x9, x21 + eor x14, x14, x17 + umulh x23, x10, x21 + adcs x13, x13, xzr + umulh x24, x11, x21 + adcs x14, x14, xzr + umulh x25, x12, x21 + adc x19, xzr, xzr // used in __smul_767x63_tail + umulh x26, x13, x21 + mul x9, x9, x21 + mul x10, x10, x21 + mul x11, x11, x21 + adds x10, x10, x22 + mul x12, x12, x21 + adcs x11, x11, x23 + mul x13, x13, x21 + adcs x12, x12, x24 + mul x28,x14, x21 + adcs x13, x13, x25 + adcs x28,x28,x26 + adc x2, x2, xzr + + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + stp x3, x4, [x0,#8*0] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*2] + adcs x27, x27, x28 + stp x7, x27, [x0,#8*4] + adc x28, x2, xzr // used in __smul_767x63_tail + + ret + + + +.align 5 +__smul_767x63_tail: + smulh x27, x8, x20 + ldp x3, x4, [x1,#8*24] // load rest of |v| + umulh x14,x14, x21 + ldp x5, x6, [x1,#8*26] + ldp x7, x8, [x1,#8*28] + + eor x3, x3, x17 // conditionally negate rest of |v| + eor x4, x4, x17 + eor x5, x5, x17 + adds x3, x3, x19 + eor x6, x6, x17 + adcs x4, x4, xzr + eor x7, x7, x17 + adcs x5, x5, xzr + eor x8, x8, x17 + adcs x6, x6, xzr + umulh x22, x3, x21 + adcs x7, x7, xzr + umulh x23, x4, x21 + adc x8, x8, xzr + + umulh x24, x5, x21 + add x14, x14, x28 + umulh x25, x6, x21 + asr x28, x27, #63 + umulh x26, x7, x21 + mul x3, x3, x21 + mul x4, x4, x21 + mul x5, x5, x21 + adds x3, x3, x14 + mul x6, x6, x21 + adcs x4, x4, x22 + mul x7, x7, x21 + adcs x5, x5, x23 + mul x8, x8, x21 + adcs x6, x6, x24 + adcs x7, x7, x25 + adc x8, x8, x26 + + adds x3, x3, x27 + adcs x4, x4, x28 + adcs x5, x5, x28 + adcs x6, x6, x28 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x28 + stp x5, x6, [x0,#8*8] + adc x8, x8, x28 + stp x7, x8, [x0,#8*10] + + ret + + + +.align 5 +__smul_383_n_shift_by_62: + ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) + asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x5, x6, [x1,#8*2+0] + eor x2, x15, x28 // conditionally negate |f0| (or |g0|) + ldp x7, x8, [x1,#8*4+0] + + eor x3, x3, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + umulh x22, x3, x2 + adcs x6, x6, xzr + umulh x23, x4, x2 + eor x8, x8, x28 + umulh x24, x5, x2 + adcs x7, x7, xzr + umulh x25, x6, x2 + adc x8, x8, xzr + + umulh x26, x7, x2 + smulh x27, x8, x2 + mul x3, x3, x2 + mul x4, x4, x2 + mul x5, x5, x2 + adds x4, x4, x22 + mul x6, x6, x2 + adcs x5, x5, x23 + mul x7, x7, x2 + adcs x6, x6, x24 + mul x8, x8, x2 + adcs x7, x7, x25 + adcs x8, x8 ,x26 + adc x27, x27, xzr + ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) + asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x11, x12, [x1,#8*2+48] + eor x2, x16, x28 // conditionally negate |f0| (or |g0|) + ldp x13, x14, [x1,#8*4+48] + + eor x9, x9, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x10, x10, x28 + adds x9, x9, x28, lsr#63 + eor x11, x11, x28 + adcs x10, x10, xzr + eor x12, x12, x28 + adcs x11, x11, xzr + eor x13, x13, x28 + umulh x22, x9, x2 + adcs x12, x12, xzr + umulh x23, x10, x2 + eor x14, x14, x28 + umulh x24, x11, x2 + adcs x13, x13, xzr + umulh x25, x12, x2 + adc x14, x14, xzr + + umulh x26, x13, x2 + smulh x28, x14, x2 + mul x9, x9, x2 + mul x10, x10, x2 + mul x11, x11, x2 + adds x10, x10, x22 + mul x12, x12, x2 + adcs x11, x11, x23 + mul x13, x13, x2 + adcs x12, x12, x24 + mul x14, x14, x2 + adcs x13, x13, x25 + adcs x14, x14 ,x26 + adc x28, x28, xzr + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x27, x28 + + extr x3, x4, x3, #62 + extr x4, x5, x4, #62 + extr x5, x6, x5, #62 + asr x28, x9, #63 + extr x6, x7, x6, #62 + extr x7, x8, x7, #62 + extr x8, x9, x8, #62 + + eor x3, x3, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + adcs x6, x6, xzr + eor x8, x8, x28 + stp x3, x4, [x0,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x0,#8*2] + adc x8, x8, xzr + stp x7, x8, [x0,#8*4] + + eor x15, x15, x28 + eor x16, x16, x28 + sub x15, x15, x28 + sub x16, x16, x28 + + ret + + +.align 4 +__ab_approximation_62: + ldp x7, x8, [x1,#8*4] + ldp x13, x14, [x1,#8*10] + ldp x5, x6, [x1,#8*2] + ldp x11, x12, [x1,#8*8] + +Lab_approximation_62_loaded: + orr x22, x8, x14 // check top-most limbs, ... + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x22, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + ldp x3, x4, [x1,#8*0] + ldp x9, x10, [x1,#8*6] + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x22, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x22, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x22, x8, x14 + csel x13, x13, x10, ne + + clz x22, x22 + cmp x22, #64 + csel x22, x22, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x23, x22 + + lslv x8, x8, x22 // align high limbs to the left + lslv x14, x14, x22 + lsrv x7, x7, x23 + lsrv x13, x13, x23 + and x7, x7, x23, asr#6 + and x13, x13, x23, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + b __inner_loop_62 + ret + + +.align 4 +__inner_loop_62: + mov x15, #1 // |f0|=1 + mov x16, #0 // |g0|=0 + mov x17, #0 // |f1|=0 + mov x19, #1 // |g1|=1 + +Loop_62: + sbfx x28, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + subs x24, x9, x3 // |b_|-|a_| + and x22, x9, x28 + sbc x25, x14, x8 + and x23, x14, x28 + subs x26, x3, x22 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x22, x15 + sbcs x27, x8, x23 + mov x23, x16 + csel x9, x9, x3, hs // |b_| = |a_| + csel x14, x14, x8, hs + csel x3, x26, x24, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x8, x27, x25, hs + csel x15, x15, x17, hs // exchange |f0| and |f1| + csel x17, x17, x22, hs + csel x16, x16, x19, hs // exchange |g0| and |g1| + csel x19, x19, x23, hs + extr x3, x8, x3, #1 + lsr x8, x8, #1 + and x22, x17, x28 + and x23, x19, x28 + add x17, x17, x17 // |f1|<<=1 + add x19, x19, x19 // |g1|<<=1 + sub x15, x15, x22 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x16, x16, x23 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, Loop_62 + + ret + diff --git a/crypto/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S b/crypto/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S new file mode 100644 index 00000000000..b5c953d287a --- /dev/null +++ b/crypto/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S @@ -0,0 +1,324 @@ +.text + +.globl _ct_is_square_mod_384 + +.align 5 +_ct_is_square_mod_384: +.long 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #512 + + ldp x3, x4, [x0,#8*0] // load input + ldp x5, x6, [x0,#8*2] + ldp x7, x8, [x0,#8*4] + + add x0, sp, #255 // find closest 256-byte-aligned spot + and x0, x0, #-256 // in the frame... + + ldp x9, x10, [x1,#8*0] // load modulus + ldp x11, x12, [x1,#8*2] + ldp x13, x14, [x1,#8*4] + + stp x3, x4, [x0,#8*6] // copy input to |a| + stp x5, x6, [x0,#8*8] + stp x7, x8, [x0,#8*10] + stp x9, x10, [x0,#8*0] // copy modulus to |b| + stp x11, x12, [x0,#8*2] + stp x13, x14, [x0,#8*4] + + eor x2, x2, x2 // init the Legendre symbol + mov x15, #24 // 24 is 768/30-1 + b Loop_is_square + +.align 4 +Loop_is_square: + bl __ab_approximation_30 + sub x15, x15, #1 + + eor x1, x0, #128 // pointer to dst |b| + bl __smul_384_n_shift_by_30 + + mov x19, x16 // |f0| + mov x20, x17 // |g0| + add x1, x1, #8*6 // pointer to dst |a| + bl __smul_384_n_shift_by_30 + + ldp x9, x10, [x1,#-8*6] + eor x0, x0, #128 // flip-flop src |a|b| + and x27, x27, x9 // if |a| was negative, + add x2, x2, x27, lsr#1 // adjust |L| + + cbnz x15, Loop_is_square + + ////////////////////////////////////////// last iteration + //bl __ab_approximation_30 // |a| and |b| are exact, + //ldr x8, [x0,#8*6] // and loaded + //ldr x14, [x0,#8*0] + mov x15, #48 // 48 is 768%30 + 30 + bl __inner_loop_48 + ldr x30, [x29,#8] + + and x0, x2, #1 + eor x0, x0, #1 + + add sp, sp, #512 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 +.long 3573752767 + ret + + + +.align 5 +__smul_384_n_shift_by_30: + ldp x3, x4, [x0,#8*0+0] // load |b| (or |a|) + asr x27, x20, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x5, x6, [x0,#8*2+0] + eor x20, x20, x27 // conditionally negate |g1| (or |f1|) + ldp x7, x8, [x0,#8*4+0] + + eor x3, x3, x27 // conditionally negate |b| (or |a|) + sub x20, x20, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + umulh x21, x3, x20 + adcs x6, x6, xzr + umulh x22, x4, x20 + eor x8, x8, x27 + umulh x23, x5, x20 + adcs x7, x7, xzr + umulh x24, x6, x20 + adc x8, x8, xzr + + umulh x25, x7, x20 + and x28, x20, x27 + umulh x26, x8, x20 + neg x28, x28 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x21 + mul x6, x6, x20 + adcs x5, x5, x22 + mul x7, x7, x20 + adcs x6, x6, x23 + mul x8, x8, x20 + adcs x7, x7, x24 + adcs x8, x8 ,x25 + adc x26, x26, x28 + ldp x9, x10, [x0,#8*0+48] // load |b| (or |a|) + asr x27, x19, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x11, x12, [x0,#8*2+48] + eor x19, x19, x27 // conditionally negate |g1| (or |f1|) + ldp x13, x14, [x0,#8*4+48] + + eor x9, x9, x27 // conditionally negate |b| (or |a|) + sub x19, x19, x27 + eor x10, x10, x27 + adds x9, x9, x27, lsr#63 + eor x11, x11, x27 + adcs x10, x10, xzr + eor x12, x12, x27 + adcs x11, x11, xzr + eor x13, x13, x27 + umulh x21, x9, x19 + adcs x12, x12, xzr + umulh x22, x10, x19 + eor x14, x14, x27 + umulh x23, x11, x19 + adcs x13, x13, xzr + umulh x24, x12, x19 + adc x14, x14, xzr + + umulh x25, x13, x19 + and x28, x19, x27 + umulh x27, x14, x19 + neg x28, x28 + mul x9, x9, x19 + mul x10, x10, x19 + mul x11, x11, x19 + adds x10, x10, x21 + mul x12, x12, x19 + adcs x11, x11, x22 + mul x13, x13, x19 + adcs x12, x12, x23 + mul x14, x14, x19 + adcs x13, x13, x24 + adcs x14, x14 ,x25 + adc x27, x27, x28 + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x26, x27 + + extr x3, x4, x3, #30 + extr x4, x5, x4, #30 + extr x5, x6, x5, #30 + asr x27, x9, #63 + extr x6, x7, x6, #30 + extr x7, x8, x7, #30 + extr x8, x9, x8, #30 + + eor x3, x3, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + adcs x6, x6, xzr + eor x8, x8, x27 + stp x3, x4, [x1,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x1,#8*2] + adc x8, x8, xzr + stp x7, x8, [x1,#8*4] + + ret + + +.align 4 +__ab_approximation_30: + ldp x13, x14, [x0,#8*4] // |a| is still in registers + ldp x11, x12, [x0,#8*2] + + orr x21, x8, x14 // check top-most limbs, ... + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x6, ne + orr x21, x8, x14 // ... ones before top-most, ... + csel x13, x13, x12, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x5, ne + orr x21, x8, x14 // ... and ones before that ... + csel x13, x13, x11, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x4, ne + orr x21, x8, x14 // and one more, ... + csel x13, x13, x10, ne + + cmp x21, #0 + csel x8, x8, x7, ne + csel x14, x14, x13, ne + csel x7, x7, x3, ne + orr x21, x8, x14 + csel x13, x13, x9, ne + + clz x21, x21 + cmp x21, #64 + csel x21, x21, xzr, ne + csel x8, x8, x7, ne + csel x14, x14, x13, ne + neg x22, x21 + + lslv x8, x8, x21 // align high limbs to the left + lslv x14, x14, x21 + lsrv x7, x7, x22 + lsrv x13, x13, x22 + and x7, x7, x22, asr#6 + and x13, x13, x22, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + bfxil x8, x3, #0, #32 + bfxil x14, x9, #0, #32 + + b __inner_loop_30 + ret + + + +.align 4 +__inner_loop_30: + mov x28, #30 + mov x17, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x20, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x27,#0x7FFFFFFF7FFFFFFF + +Loop_30: + sbfx x24, x8, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x8, x14 + sub x28, x28, #1 + and x21, x14, x24 + + sub x22, x14, x8 // |b_|-|a_| + subs x23, x8, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 // L + (a_ & b_) >> 1 + mov x21, x20 + csel x14, x14, x8, hs // |b_| = |a_| + csel x8, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x20, x20, x17, hs // exchange |fg0| and |fg1| + csel x17, x17, x21, hs + csel x2, x2, x25, hs + lsr x8, x8, #1 + and x21, x20, x24 + and x22, x27, x24 + add x23, x14, #2 + sub x17, x17, x21 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x20, x20, x20 // |f1|<<=1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + add x17, x17, x22 + sub x20, x20, x27 + + cbnz x28, Loop_30 + + mov x27, #0x7FFFFFFF + ubfx x16, x17, #0, #32 + ubfx x17, x17, #32, #32 + ubfx x19, x20, #0, #32 + ubfx x20, x20, #32, #32 + sub x16, x16, x27 // remove the bias + sub x17, x17, x27 + sub x19, x19, x27 + sub x20, x20, x27 + + ret + + +.align 4 +__inner_loop_48: +Loop_48: + sbfx x24, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x3, x9 + sub x15, x15, #1 + and x21, x9, x24 + sub x22, x9, x3 // |b_|-|a_| + subs x23, x3, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 + csel x9, x9, x3, hs // |b_| = |a_| + csel x3, x23, x22, hs // borrow means |a_|<|b_|, replace with |b_|-|a_| + csel x2, x2, x25, hs + add x23, x9, #2 + lsr x3, x3, #1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + + cbnz x15, Loop_48 + + ret + diff --git a/crypto/blst_src/build/mach-o/ct_is_square_mod_384-x86_64.s b/crypto/blst_src/build/mach-o/ct_is_square_mod_384-x86_64.s new file mode 100644 index 00000000000..f2823941167 --- /dev/null +++ b/crypto/blst_src/build/mach-o/ct_is_square_mod_384-x86_64.s @@ -0,0 +1,471 @@ +.text + +.globl _ct_is_square_mod_384 + +.p2align 5 +_ct_is_square_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $536,%rsp +.cfi_adjust_cfa_offset 536 + + + leaq 24+255(%rsp),%rax + andq $-256,%rax + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbx + movq 24(%rsi),%rcx + movq 32(%rsi),%rdx + movq 40(%rsi),%rdi + movq %rax,%rsi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rcx,72(%rax) + movq %rdx,80(%rax) + movq %rdi,88(%rax) + + xorq %rbp,%rbp + movl $24,%ecx + jmp L$oop_is_square + +.p2align 5 +L$oop_is_square: + movl %ecx,16(%rsp) + + call __ab_approximation_30 + movq %rax,0(%rsp) + movq %rbx,8(%rsp) + + movq $128+48,%rdi + xorq %rsi,%rdi + call __smulq_384_n_shift_by_30 + + movq 0(%rsp),%rdx + movq 8(%rsp),%rcx + leaq -48(%rdi),%rdi + call __smulq_384_n_shift_by_30 + + movl 16(%rsp),%ecx + xorq $128,%rsi + + andq 48(%rdi),%r14 + shrq $1,%r14 + addq %r14,%rbp + + subl $1,%ecx + jnz L$oop_is_square + + + + + movq 48(%rsi),%r9 + call __inner_loop_48 + + movq $1,%rax + andq %rbp,%rax + xorq $1,%rax + + leaq 536(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -536-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__smulq_384_n_shift_by_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r14 + andq %rbx,%r14 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r14 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r14 + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbx + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbx + addq %rax,%rbx + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %rdx,%r15 + andq %rbx,%r15 + mulq %rbx + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbx + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbx + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbx + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbx + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + negq %r15 + mulq %rbx + addq %rax,%r13 + adcq %rdx,%r15 + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %r15,%r14 + + shrdq $30,%r9,%r8 + shrdq $30,%r10,%r9 + shrdq $30,%r11,%r10 + shrdq $30,%r12,%r11 + shrdq $30,%r13,%r12 + shrdq $30,%r14,%r13 + + sarq $63,%r14 + xorq %rbx,%rbx + subq %r14,%rbx + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbx,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__ab_approximation_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 88(%rsi),%rbx + movq 80(%rsi),%r15 + movq 72(%rsi),%r14 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r11,%r12 + movq 64(%rsi),%r11 + cmovzq %r14,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r10,%r12 + movq 56(%rsi),%r10 + cmovzq %r11,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r9,%r12 + movq 48(%rsi),%r9 + cmovzq %r10,%r15 + + movq %r13,%rax + orq %rbx,%rax + cmovzq %r12,%r13 + cmovzq %r15,%rbx + cmovzq %r8,%r12 + cmovzq %r9,%r15 + + movq %r13,%rax + orq %rbx,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r13 + cmovzq %r9,%rbx + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%r12,%r13 + shldq %cl,%r15,%rbx + + movq $0xFFFFFFFF00000000,%rax + movl %r8d,%r8d + movl %r9d,%r9d + andq %rax,%r13 + andq %rax,%rbx + orq %r13,%r8 + orq %rbx,%r9 + + jmp __inner_loop_30 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__inner_loop_30: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rbx + movq $0x800000007FFFFFFF,%rcx + leaq -1(%rbx),%r15 + movl $30,%edi + +L$oop_30: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbx,%r12 + movq %rcx,%r13 + movq %rbp,%r14 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rcx,%rbx + cmovbq %r12,%rcx + cmovbq %rax,%rbp + + subq %r9,%r8 + subq %rcx,%rbx + addq %r15,%rbx + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbx + cmovzq %r13,%rcx + cmovzq %r14,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rcx,%rcx + leaq (%rax,%rbp,1),%rbp + subq %r15,%rcx + + subl $1,%edi + jnz L$oop_30 + + shrq $32,%r15 + movl %ebx,%eax + shrq $32,%rbx + movl %ecx,%edx + shrq $32,%rcx + subq %r15,%rax + subq %r15,%rbx + subq %r15,%rdx + subq %r15,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__inner_loop_48: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movl $48,%edi + +L$oop_48: + movq %r8,%rax + andq %r9,%rax + shrq $1,%rax + + cmpq %r9,%r8 + movq %r8,%r10 + movq %r9,%r11 + leaq (%rax,%rbp,1),%rax + movq %rbp,%r12 + cmovbq %r9,%r8 + cmovbq %r10,%r9 + cmovbq %rax,%rbp + + subq %r9,%r8 + + testq $1,%r10 + cmovzq %r10,%r8 + cmovzq %r11,%r9 + cmovzq %r12,%rbp + + leaq 2(%r9),%rax + shrq $1,%r8 + shrq $2,%rax + addq %rax,%rbp + + subl $1,%edi + jnz L$oop_48 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s b/crypto/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..185a876b87c --- /dev/null +++ b/crypto/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s @@ -0,0 +1,1187 @@ +.text + +.globl _ct_inverse_mod_383 + +.p2align 5 +_ct_inverse_mod_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1112,%rsp +.cfi_adjust_cfa_offset 1112 + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + xorq $256+96,%rsi + movl $62,%edi + call __ab_approximation_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_383_n_shift_by_62 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + xorq $256+96,%rsi + movl $62,%edi + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 48(%rsi),%r10 + movq 56(%rsi),%r11 + call __inner_loop_62 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + movq %r8,0(%rdi) + movq %r10,48(%rdi) + + + + leaq 96(%rsi),%rsi + leaq 96(%rdi),%rdi + call __smulq_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulq_767x63 + + + xorq $256+96,%rsi + movl $22,%edi + + movq 0(%rsi),%r8 + xorq %r9,%r9 + movq 48(%rsi),%r10 + xorq %r11,%r11 + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulq_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1112-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulq_767x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,0(%rdi) + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + movq %r9,8(%rdi) + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + movq %r10,16(%rdi) + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %r11,24(%rdi) + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + movq %r12,32(%rdi) + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + movq %r13,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + movq %rdx,%rsi + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rsi + addq %rax,%rsi + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + xorq %rdx,%r14 + xorq %rdx,%r15 + xorq %rdx,%rbx + xorq %rdx,%rbp + xorq %rdx,%rcx + xorq %rdx,%rdi + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulq %rsi + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rsi + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rsi + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rsi + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rsi + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r15 + mulq %rsi + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %rdx,%rbx + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + mulq %rsi + addq %rax,%rbp + movq %rcx,%rax + adcq $0,%rdx + movq %rdx,%rcx + mulq %rsi + addq %rax,%rcx + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%rdi + movq 8(%rsp),%rdx + imulq %rsi,%rax + movq 16(%rsp),%rsi + addq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulq_383x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq 48(%rsi),%rsi + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp,%rax + addq %rax,%r13 + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulq_383_n_shift_by_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq 48(%rsi),%rsi + movq %rdx,%r14 + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rdx + xorq %rax,%rax + subq %rdx,%rax + + xorq %rdx,%rbp + addq %rax,%rbp + + xorq %rdx,%r8 + xorq %rdx,%r9 + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %r8,%rax + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulq %rbp + movq %rax,%r8 + movq %r9,%rax + movq %rdx,%r9 + mulq %rbp + addq %rax,%r9 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r10 + mulq %rbp + addq %rax,%r10 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r11 + mulq %rbp + addq %rax,%r11 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r12 + mulq %rbp + addq %rax,%r12 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r13 + imulq %rbp + addq %rax,%r13 + adcq $0,%rdx + + leaq -48(%rsi),%rsi + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $62,%r9,%r8 + shrdq $62,%r10,%r9 + shrdq $62,%r11,%r10 + shrdq $62,%r12,%r11 + shrdq $62,%r13,%r12 + shrdq $62,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__ab_approximation_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 16(%rsi),%r8 + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 8(%rsi),%r8 + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + movq 0(%rsi),%r8 + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + jmp __inner_loop_62 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 3 +.long 0 +__inner_loop_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + movq %rsi,8(%rsp) + +L$oop_62: + xorq %rax,%rax + xorq %rbx,%rbx + testq $1,%r8 + movq %r10,%rbp + movq %r11,%r14 + cmovnzq %r10,%rax + cmovnzq %r11,%rbx + subq %r8,%rbp + sbbq %r9,%r14 + movq %r8,%r15 + movq %r9,%rsi + subq %rax,%r8 + sbbq %rbx,%r9 + cmovcq %rbp,%r8 + cmovcq %r14,%r9 + cmovcq %r15,%r10 + cmovcq %rsi,%r11 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrdq $1,%r9,%r8 + shrq $1,%r9 + testq $1,%r15 + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz L$oop_62 + + movq 8(%rsp),%rsi + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s b/crypto/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s new file mode 100644 index 00000000000..3e05df3a4b3 --- /dev/null +++ b/crypto/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s @@ -0,0 +1,1566 @@ +.text + +.globl _ctx_inverse_mod_383 + +.p2align 5 +_ctx_inverse_mod_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $1112,%rsp +.cfi_adjust_cfa_offset 1112 + + + leaq 88+511(%rsp),%rax + andq $-512,%rax + movq %rdi,32(%rsp) + movq %rcx,40(%rsp) + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq 0(%rdx),%r14 + movq 8(%rdx),%r15 + movq 16(%rdx),%rbx + movq 24(%rdx),%rbp + movq 32(%rdx),%rsi + movq 40(%rdx),%rdi + + movq %r8,0(%rax) + movq %r9,8(%rax) + movq %r10,16(%rax) + movq %r11,24(%rax) + movq %r12,32(%rax) + movq %r13,40(%rax) + + movq %r14,48(%rax) + movq %r15,56(%rax) + movq %rbx,64(%rax) + movq %rbp,72(%rax) + movq %rsi,80(%rax) + movq %rax,%rsi + movq %rdi,88(%rax) + + + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + movq %rdx,96(%rdi) + + + xorq $256,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + + + + movq 96(%rsi),%rax + movq 144(%rsi),%r11 + movq %rdx,%rbx + movq %rax,%r10 + imulq 56(%rsp) + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq 64(%rsp) + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + sarq $63,%r9 + movq %r9,64(%rdi) + movq %r9,72(%rdi) + movq %r9,80(%rdi) + movq %r9,88(%rdi) + leaq 96(%rsi),%rsi + + movq %r10,%rax + imulq %rbx + movq %rax,%r8 + movq %r11,%rax + movq %rdx,%r9 + imulq %rcx + addq %rax,%r8 + adcq %rdx,%r9 + movq %r8,96(%rdi) + movq %r9,104(%rdi) + sarq $63,%r9 + movq %r9,112(%rdi) + movq %r9,120(%rdi) + movq %r9,128(%rdi) + movq %r9,136(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383x63 + sarq $63,%r13 + movq %r13,48(%rdi) + movq %r13,56(%rdi) + movq %r13,64(%rdi) + movq %r13,72(%rdi) + movq %r13,80(%rdi) + movq %r13,88(%rdi) + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_383_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + xorq $256+96,%rsi + movl $31,%edi + call __ab_approximation_31 + + + movq %r12,72(%rsp) + movq %r13,80(%rsp) + + movq $256,%rdi + xorq %rsi,%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,56(%rsp) + movq %rcx,64(%rsp) + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_191_n_shift_by_31 + movq %rdx,72(%rsp) + movq %rcx,80(%rsp) + + movq 56(%rsp),%rdx + movq 64(%rsp),%rcx + leaq 96(%rsi),%rsi + leaq 48(%rdi),%rdi + call __smulx_383x63 + + movq 72(%rsp),%rdx + movq 80(%rsp),%rcx + leaq 48(%rdi),%rdi + call __smulx_767x63 + + xorq $256+96,%rsi + movl $53,%edi + + movq 0(%rsi),%r8 + + movq 48(%rsi),%r10 + + call __inner_loop_62 + + + + + + + + leaq 96(%rsi),%rsi + + + + + + movq %r12,%rdx + movq %r13,%rcx + movq 32(%rsp),%rdi + call __smulx_767x63 + + movq 40(%rsp),%rsi + movq %rax,%rdx + sarq $63,%rax + + movq %rax,%r8 + movq %rax,%r9 + movq %rax,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %rax,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %rax,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%rax + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %rax,%rdx + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rdx,88(%rdi) + + leaq 1112(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -1112-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulx_767x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + leaq 48(%rsi),%rsi + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq %rdx,48(%rdi) + sarq $63,%rdx + movq %rdx,56(%rdi) + movq %rcx,%rdx + movq %rcx,%rax + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + movq 56(%rsi),%r15 + movq 64(%rsi),%rbx + movq 72(%rsi),%rbp + movq 80(%rsi),%rcx + movq 88(%rsi),%rdi + + sarq $63,%rax + xorq %rsi,%rsi + subq %rax,%rsi + + xorq %rax,%rdx + addq %rsi,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %rax,%r13 + xorq %rax,%r14 + xorq %rax,%r15 + xorq %rax,%rbx + xorq %rax,%rbp + xorq %rax,%rcx + xorq %rax,%rdi + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + adcq $0,%r14 + adcq $0,%r15 + adcq $0,%rbx + adcq $0,%rbp + adcq $0,%rcx + adcq $0,%rdi + + mulxq %r8,%r8,%rax + mulxq %r9,%r9,%rsi + addq %rax,%r9 + mulxq %r10,%r10,%rax + adcq %rsi,%r10 + mulxq %r11,%r11,%rsi + adcq %rax,%r11 + mulxq %r12,%r12,%rax + adcq %rsi,%r12 + mulxq %r13,%r13,%rsi + adcq %rax,%r13 + mulxq %r14,%r14,%rax + adcq %rsi,%r14 + mulxq %r15,%r15,%rsi + adcq %rax,%r15 + mulxq %rbx,%rbx,%rax + adcq %rsi,%rbx + mulxq %rbp,%rbp,%rsi + adcq %rax,%rbp + mulxq %rcx,%rcx,%rax + adcq %rsi,%rcx + mulxq %rdi,%rdi,%rsi + movq 8(%rsp),%rdx + movq 16(%rsp),%rsi + adcq %rdi,%rax + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + adcq 24(%rdx),%r11 + adcq 32(%rdx),%r12 + adcq 40(%rdx),%r13 + adcq 48(%rdx),%r14 + movq 56(%rdx),%rdi + adcq %rdi,%r15 + adcq %rdi,%rbx + adcq %rdi,%rbp + adcq %rdi,%rcx + adcq %rdi,%rax + + movq %rdx,%rdi + + movq %r8,0(%rdx) + movq %r9,8(%rdx) + movq %r10,16(%rdx) + movq %r11,24(%rdx) + movq %r12,32(%rdx) + movq %r13,40(%rdx) + movq %r14,48(%rdx) + movq %r15,56(%rdx) + movq %rbx,64(%rdx) + movq %rbp,72(%rdx) + movq %rcx,80(%rdx) + movq %rax,88(%rdx) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulx_383x63: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + movq %rcx,%rdx + adcq %rbp,%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rbp + sarq $63,%rbp + xorq %rax,%rax + subq %rbp,%rax + + xorq %rbp,%rdx + addq %rax,%rdx + + xorq %rbp,%r8 + xorq %rbp,%r9 + xorq %rbp,%r10 + xorq %rbp,%r11 + xorq %rbp,%r12 + xorq %rbp,%r13 + addq %rax,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%rax + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %rax,%r10 + mulxq %r11,%r11,%rax + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulx_383_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + xorq %r14,%r14 + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + movq 0+24(%rsi),%r11 + movq 0+32(%rsi),%r12 + movq 0+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq %rdx,%r14 + + movq %rcx,%rdx + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + movq 48+0(%rsi),%r8 + movq 48+8(%rsi),%r9 + movq 48+16(%rsi),%r10 + movq 48+24(%rsi),%r11 + movq 48+32(%rsi),%r12 + movq 48+40(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %rax,%r10 + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r13 + addq %rbp,%r9 + mulxq %r10,%r10,%rbp + adcq %r13,%r10 + mulxq %r11,%r11,%r13 + adcq %rbp,%r11 + mulxq %r12,%r12,%rbp + adcq %r13,%r12 + adcq $0,%rbp + imulq %rdx + addq %rbp,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%rax + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r9,%r8 + shrdq $31,%r10,%r9 + shrdq $31,%r11,%r10 + shrdq $31,%r12,%r11 + shrdq $31,%rax,%r12 + shrdq $31,%r14,%rax + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r8 + xorq %r14,%r9 + xorq %r14,%r10 + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%rax + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %rax,40(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__smulx_191_n_shift_by_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rbx + movq 0+0(%rsi),%r8 + movq 0+8(%rsi),%r9 + movq 0+16(%rsi),%r10 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r8 + xorq %rax,%r9 + xorq %r10,%rax + addq %rbp,%r8 + adcq $0,%r9 + adcq $0,%rax + + mulxq %r8,%r8,%rbp + mulxq %r9,%r9,%r10 + addq %rbp,%r9 + adcq $0,%r10 + imulq %rdx + addq %rax,%r10 + adcq $0,%rdx + movq %rdx,%r14 + movq %rcx,%rdx + movq 48+0(%rsi),%r11 + movq 48+8(%rsi),%r12 + movq 48+16(%rsi),%r13 + + movq %rdx,%rax + sarq $63,%rax + xorq %rbp,%rbp + subq %rax,%rbp + + xorq %rax,%rdx + addq %rbp,%rdx + + xorq %rax,%r11 + xorq %rax,%r12 + xorq %r13,%rax + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%rax + + mulxq %r11,%r11,%rbp + mulxq %r12,%r12,%r13 + addq %rbp,%r12 + adcq $0,%r13 + imulq %rdx + addq %rax,%r13 + adcq $0,%rdx + addq %r8,%r11 + adcq %r9,%r12 + adcq %r10,%r13 + adcq %rdx,%r14 + movq %rbx,%rdx + + shrdq $31,%r12,%r11 + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 + + sarq $63,%r14 + xorq %rbp,%rbp + subq %r14,%rbp + + xorq %r14,%r11 + xorq %r14,%r12 + xorq %r14,%r13 + addq %rbp,%r11 + adcq $0,%r12 + adcq $0,%r13 + + movq %r11,0(%rdi) + movq %r12,8(%rdi) + movq %r13,16(%rdi) + + xorq %r14,%rdx + xorq %r14,%rcx + addq %rbp,%rdx + addq %rbp,%rcx + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__ab_approximation_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 40(%rsi),%r9 + movq 88(%rsi),%r11 + movq 32(%rsi),%rbx + movq 80(%rsi),%rbp + movq 24(%rsi),%r8 + movq 72(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 16(%rsi),%r8 + cmovzq %r10,%rbp + movq 64(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 8(%rsi),%r8 + cmovzq %r10,%rbp + movq 56(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + movq 0(%rsi),%r8 + cmovzq %r10,%rbp + movq 48(%rsi),%r10 + + movq %r9,%rax + orq %r11,%rax + cmovzq %rbx,%r9 + cmovzq %rbp,%r11 + cmovzq %r8,%rbx + cmovzq %r10,%rbp + + movq %r9,%rax + orq %r11,%rax + bsrq %rax,%rcx + leaq 1(%rcx),%rcx + cmovzq %r8,%r9 + cmovzq %r10,%r11 + cmovzq %rax,%rcx + negq %rcx + + + shldq %cl,%rbx,%r9 + shldq %cl,%rbp,%r11 + + movl $0x7FFFFFFF,%eax + andq %rax,%r8 + andq %rax,%r10 + andnq %r9,%rax,%r9 + andnq %r11,%rax,%r11 + orq %r9,%r8 + orq %r11,%r10 + + jmp __inner_loop_31 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__inner_loop_31: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $0x7FFFFFFF80000000,%rcx + movq $0x800000007FFFFFFF,%r13 + movq $0x7FFFFFFF7FFFFFFF,%r15 + +L$oop_31: + cmpq %r10,%r8 + movq %r8,%rax + movq %r10,%rbx + movq %rcx,%rbp + movq %r13,%r14 + cmovbq %r10,%r8 + cmovbq %rax,%r10 + cmovbq %r13,%rcx + cmovbq %rbp,%r13 + + subq %r10,%r8 + subq %r13,%rcx + addq %r15,%rcx + + testq $1,%rax + cmovzq %rax,%r8 + cmovzq %rbx,%r10 + cmovzq %rbp,%rcx + cmovzq %r14,%r13 + + shrq $1,%r8 + addq %r13,%r13 + subq %r15,%r13 + subl $1,%edi + jnz L$oop_31 + + shrq $32,%r15 + movl %ecx,%edx + movl %r13d,%r12d + shrq $32,%rcx + shrq $32,%r13 + subq %r15,%rdx + subq %r15,%rcx + subq %r15,%r12 + subq %r15,%r13 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__inner_loop_62: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq $1,%rdx + xorq %rcx,%rcx + xorq %r12,%r12 + movq $1,%r13 + +L$oop_62: + xorq %rax,%rax + testq $1,%r8 + movq %r10,%rbx + cmovnzq %r10,%rax + subq %r8,%rbx + movq %r8,%rbp + subq %rax,%r8 + cmovcq %rbx,%r8 + cmovcq %rbp,%r10 + movq %rdx,%rax + cmovcq %r12,%rdx + cmovcq %rax,%r12 + movq %rcx,%rbx + cmovcq %r13,%rcx + cmovcq %rbx,%r13 + xorq %rax,%rax + xorq %rbx,%rbx + shrq $1,%r8 + testq $1,%rbp + cmovnzq %r12,%rax + cmovnzq %r13,%rbx + addq %r12,%r12 + addq %r13,%r13 + subq %rax,%rdx + subq %rbx,%rcx + subl $1,%edi + jnz L$oop_62 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/div3w-armv8.S b/crypto/blst_src/build/mach-o/div3w-armv8.S new file mode 100644 index 00000000000..5a5eb3a01d7 --- /dev/null +++ b/crypto/blst_src/build/mach-o/div3w-armv8.S @@ -0,0 +1,88 @@ +.text + +.globl _div_3_limbs + +.align 5 +_div_3_limbs: + ldp x4,x5,[x0] // load R + eor x0,x0,x0 // Q = 0 + mov x3,#64 // loop counter + nop + +Loop: + subs x6,x4,x1 // R - D + add x0,x0,x0 // Q <<= 1 + sbcs x7,x5,x2 + add x0,x0,#1 // Q + speculative bit + csel x4,x4,x6,lo // select between R and R - D + extr x1,x2,x1,#1 // D >>= 1 + csel x5,x5,x7,lo + lsr x2,x2,#1 + sbc x0,x0,xzr // subtract speculative bit + sub x3,x3,#1 + cbnz x3,Loop + + asr x3,x0,#63 // top bit -> mask + add x0,x0,x0 // Q <<= 1 + subs x6,x4,x1 // R - D + add x0,x0,#1 // Q + specilative bit + sbcs x7,x5,x2 + sbc x0,x0,xzr // subtract speculative bit + + orr x0,x0,x3 // all ones if overflow + + ret + +.globl _quot_rem_128 + +.align 5 +_quot_rem_128: + ldp x3,x4,[x1] + + mul x5,x3,x2 // divisor[0:1} * quotient + umulh x6,x3,x2 + mul x11, x4,x2 + umulh x7,x4,x2 + + ldp x8,x9,[x0] // load 3 limbs of the dividend + ldr x10,[x0,#16] + + adds x6,x6,x11 + adc x7,x7,xzr + + subs x8,x8,x5 // dividend - divisor * quotient + sbcs x9,x9,x6 + sbcs x10,x10,x7 + sbc x5,xzr,xzr // borrow -> mask + + add x2,x2,x5 // if borrowed, adjust the quotient ... + and x3,x3,x5 + and x4,x4,x5 + adds x8,x8,x3 // ... and add divisor + adc x9,x9,x4 + + stp x8,x9,[x0] // save 2 limbs of the remainder + str x2,[x0,#16] // and one limb of the quotient + + mov x0,x2 // return adjusted quotient + + ret + + +.globl _quot_rem_64 + +.align 5 +_quot_rem_64: + ldr x3,[x1] + ldr x8,[x0] // load 1 limb of the dividend + + mul x5,x3,x2 // divisor * quotient + + sub x8,x8,x5 // dividend - divisor * quotient + + stp x8,x2,[x0] // save remainder and quotient + + mov x0,x2 // return quotient + + ret + diff --git a/crypto/blst_src/build/mach-o/div3w-x86_64.s b/crypto/blst_src/build/mach-o/div3w-x86_64.s new file mode 100644 index 00000000000..8075571c87d --- /dev/null +++ b/crypto/blst_src/build/mach-o/div3w-x86_64.s @@ -0,0 +1,115 @@ +.text + +.globl _div_3_limbs +.private_extern _div_3_limbs + +.p2align 5 +_div_3_limbs: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq (%rdi),%r8 + movq 8(%rdi),%r9 + xorq %rax,%rax + movl $64,%ecx + +L$oop: + movq %r8,%r10 + subq %rsi,%r8 + movq %r9,%r11 + sbbq %rdx,%r9 + leaq 1(%rax,%rax,1),%rax + movq %rdx,%rdi + cmovcq %r10,%r8 + cmovcq %r11,%r9 + sbbq $0,%rax + shlq $63,%rdi + shrq $1,%rsi + shrq $1,%rdx + orq %rdi,%rsi + subl $1,%ecx + jnz L$oop + + leaq 1(%rax,%rax,1),%rcx + sarq $63,%rax + + subq %rsi,%r8 + sbbq %rdx,%r9 + sbbq $0,%rcx + + orq %rcx,%rax + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _quot_rem_128 +.private_extern _quot_rem_128 + +.p2align 5 +_quot_rem_128: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rax + movq %rdx,%rcx + + mulq 0(%rsi) + movq %rax,%r8 + movq %rcx,%rax + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r9 + adcq $0,%rdx + + movq 0(%rdi),%r10 + movq 8(%rdi),%r11 + movq 16(%rdi),%rax + + subq %r8,%r10 + sbbq %r9,%r11 + sbbq %rdx,%rax + sbbq %r8,%r8 + + addq %r8,%rcx + movq %r8,%r9 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + addq %r8,%r10 + adcq %r9,%r11 + + movq %r10,0(%rdi) + movq %r11,8(%rdi) + movq %rcx,16(%rdi) + + movq %rcx,%rax + + .byte 0xf3,0xc3 +.cfi_endproc + + + + + + +.globl _quot_rem_64 +.private_extern _quot_rem_64 + +.p2align 5 +_quot_rem_64: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rdx,%rax + imulq 0(%rsi),%rdx + + movq 0(%rdi),%r10 + + subq %rdx,%r10 + + movq %r10,0(%rdi) + movq %rax,8(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/mul_mont_256-armv8.S b/crypto/blst_src/build/mach-o/mul_mont_256-armv8.S new file mode 100644 index 00000000000..4f506b58b0f --- /dev/null +++ b/crypto/blst_src/build/mach-o/mul_mont_256-armv8.S @@ -0,0 +1,464 @@ +.text + +.globl _mul_mont_sparse_256 +.private_extern _mul_mont_sparse_256 + +.align 5 +_mul_mont_sparse_256: + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x10,x11,[x1] + ldr x9, [x2] + ldp x12,x13,[x1,#16] + + mul x19,x10,x9 + ldp x5,x6,[x3] + mul x20,x11,x9 + ldp x7,x8,[x3,#16] + mul x21,x12,x9 + mul x22,x13,x9 + + umulh x14,x10,x9 + umulh x15,x11,x9 + mul x3,x4,x19 + umulh x16,x12,x9 + umulh x17,x13,x9 + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,xzr, x17 + mul x17,x8,x3 + ldr x9,[x2,8*1] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*2] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*3] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + adcs x20,x21,x15 + adcs x21,x22,x16 + adcs x22,x23,x17 + adc x23,xzr,xzr + + subs x14,x19,x5 + sbcs x15,x20,x6 + sbcs x16,x21,x7 + sbcs x17,x22,x8 + sbcs xzr, x23,xzr + + csel x19,x19,x14,lo + csel x20,x20,x15,lo + csel x21,x21,x16,lo + csel x22,x22,x17,lo + + stp x19,x20,[x0] + stp x21,x22,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + ret + +.globl _sqr_mont_sparse_256 +.private_extern _sqr_mont_sparse_256 + +.align 5 +_sqr_mont_sparse_256: +.long 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + mov x4,x3 + + //////////////////////////////////////////////////////////////// + // | | | | | |a1*a0| | + // | | | | |a2*a0| | | + // | |a3*a2|a3*a0| | | | + // | | | |a2*a1| | | | + // | | |a3*a1| | | | | + // *| | | | | | | | 2| + // +|a3*a3|a2*a2|a1*a1|a0*a0| + // |--+--+--+--+--+--+--+--| + // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is x10 + // + // "can't overflow" below mark carrying into high part of + // multiplication result, which can't overflow, because it + // can never be all ones. + + mul x11,x6,x5 // a[1]*a[0] + umulh x15,x6,x5 + mul x12,x7,x5 // a[2]*a[0] + umulh x16,x7,x5 + mul x13,x8,x5 // a[3]*a[0] + umulh x19,x8,x5 + + adds x12,x12,x15 // accumulate high parts of multiplication + mul x14,x7,x6 // a[2]*a[1] + umulh x15,x7,x6 + adcs x13,x13,x16 + mul x16,x8,x6 // a[3]*a[1] + umulh x17,x8,x6 + adc x19,x19,xzr // can't overflow + + mul x20,x8,x7 // a[3]*a[2] + umulh x21,x8,x7 + + adds x15,x15,x16 // accumulate high parts of multiplication + mul x10,x5,x5 // a[0]*a[0] + adc x16,x17,xzr // can't overflow + + adds x13,x13,x14 // accumulate low parts of multiplication + umulh x5,x5,x5 + adcs x19,x19,x15 + mul x15,x6,x6 // a[1]*a[1] + adcs x20,x20,x16 + umulh x6,x6,x6 + adc x21,x21,xzr // can't overflow + + adds x11,x11,x11 // acc[1-6]*=2 + mul x16,x7,x7 // a[2]*a[2] + adcs x12,x12,x12 + umulh x7,x7,x7 + adcs x13,x13,x13 + mul x17,x8,x8 // a[3]*a[3] + adcs x19,x19,x19 + umulh x8,x8,x8 + adcs x20,x20,x20 + adcs x21,x21,x21 + adc x22,xzr,xzr + + adds x11,x11,x5 // +a[i]*a[i] + adcs x12,x12,x15 + adcs x13,x13,x6 + adcs x19,x19,x16 + adcs x20,x20,x7 + adcs x21,x21,x17 + adc x22,x22,x8 + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + adds x10,x10,x19 // accumulate upper half + adcs x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adc x19,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x19,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 +.long 3573752767 + ret + +.globl _from_mont_256 +.private_extern _from_mont_256 + +.align 5 +_from_mont_256: +.long 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 +.long 3573752767 + ret + + +.globl _redc_mont_256 +.private_extern _redc_mont_256 + +.align 5 +_redc_mont_256: +.long 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + ldp x14,x15,[x1,#32] + ldp x16,x17,[x1,#48] + + adds x10,x10,x14 + adcs x11,x11,x15 + adcs x12,x12,x16 + adcs x13,x13,x17 + adc x9,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x9,xzr + + csel x10,x10,x14,lo + csel x11,x11,x15,lo + csel x12,x12,x16,lo + csel x13,x13,x17,lo + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 +.long 3573752767 + ret + + + +.align 5 +__mul_by_1_mont_256: + mul x3,x4,x10 + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + adc x13,x9,x17 + + ret + diff --git a/crypto/blst_src/build/mach-o/mul_mont_384-armv8.S b/crypto/blst_src/build/mach-o/mul_mont_384-armv8.S new file mode 100644 index 00000000000..5aa2e9f3ae7 --- /dev/null +++ b/crypto/blst_src/build/mach-o/mul_mont_384-armv8.S @@ -0,0 +1,2372 @@ +.text + +.globl _add_mod_384x384 + +.align 5 +_add_mod_384x384: +.long 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __add_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 +.long 3573752767 + ret + + + +.align 5 +__add_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + stp x11, x12, [x0] + adcs x15,x15,x23 + ldp x11, x12, [x1,#48] + adcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + adcs x11,x11,x19 + stp x15, x16, [x0,#32] + adcs x12,x12,x20 + ldp x15, x16, [x1,#80] + adcs x13,x13,x21 + ldp x23,x24,[x2,#80] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + stp x11,x12,[x0,#48] + csel x15,x15,x23,lo + stp x13,x14,[x0,#64] + csel x16,x16,x24,lo + stp x15,x16,[x0,#80] + + ret + + +.globl _sub_mod_384x384 + +.align 5 +_sub_mod_384x384: +.long 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 +.long 3573752767 + ret + + + +.align 5 +__sub_mod_384x384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + stp x11, x12, [x0] + sbcs x15,x15,x23 + ldp x11, x12, [x1,#48] + sbcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + sbcs x11,x11,x19 + stp x15, x16, [x0,#32] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#80] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#80] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + ret + + + +.align 5 +__add_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + stp x11,x12,[x0] + csel x16,x16,x24,lo + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + + +.align 5 +__sub_mod_384: + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0] + adc x16,x16,x24 + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + +.globl _mul_mont_384x +.private_extern _mul_mont_384x + +.align 5 +_mul_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#288 // space for 3 768-bit vectors + + mov x26,x0 // save r_ptr + mov x27,x1 // save b_ptr + mov x28,x2 // save b_ptr + + sub x0,sp,#0 // mul_384(t0, a->re, b->re) + bl __mul_384 + + add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x2,x2,#48 + add x0,sp,#96 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + sub x2,x1,#48 + add x0,sp,#240 + bl __add_mod_384 + + add x1,x28,#0 + add x2,x28,#48 + add x0,sp,#192 // t2 + bl __add_mod_384 + + add x1,x0,#0 + add x2,x0,#48 + bl __mul_384 // mul_384(t2, a->re+a->im, b->re+b->im) + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,x0 + add x2,sp,#0 + bl __sub_mod_384x384 + + add x2,sp,#96 + bl __sub_mod_384x384 // t2 = t2-t0-t1 + + add x1,sp,#0 + add x2,sp,#96 + add x0,sp,#0 + bl __sub_mod_384x384 // t0 = t0-t1 + + add x1,sp,#0 // ret->re = redc(t0) + add x0,x26,#0 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + + add x1,sp,#192 // ret->im = redc(t2) + add x0,x0,#48 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#288 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _sqr_mont_384x +.private_extern _sqr_mont_384x + +.align 5 +_sqr_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 2 384-bit vectors + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + add x0,sp,#0 + bl __add_mod_384 // t0 = a->re + a->im + + add x0,sp,#48 + bl __sub_mod_384 // t1 = a->re - a->im + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __mul_mont_384 // mul_mont_384(ret->im, a->re, a->im) + + adds x11,x11,x11 // add with itself + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x19,x11,x19,lo + csel x20,x12,x20,lo + csel x21,x13,x21,lo + ldp x11,x12,[sp] + csel x22,x14,x22,lo + ldr x17, [sp,#48] + csel x23,x15,x23,lo + ldp x13,x14,[sp,#16] + csel x24,x16,x24,lo + ldp x15,x16,[sp,#32] + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + add x2,sp,#48 + bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _mul_mont_384 +.private_extern _mul_mont_384 + +.align 5 +_mul_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + + +.align 5 +__mul_mont_384: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + mov x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*1] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*2] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*3] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*4] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*5] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + adc x17,x17,xzr + + adds x19,x20,x26 + adcs x20,x21,x27 + adcs x21,x22,x28 + adcs x22,x23,x0 + adcs x23,x24,x1 + adcs x24,x25,x3 + adc x25,x17,xzr + + subs x26,x19,x5 + sbcs x27,x20,x6 + sbcs x28,x21,x7 + sbcs x0,x22,x8 + sbcs x1,x23,x9 + sbcs x3,x24,x10 + sbcs xzr, x25,xzr + + csel x11,x19,x26,lo + csel x12,x20,x27,lo + csel x13,x21,x28,lo + csel x14,x22,x0,lo + csel x15,x23,x1,lo + csel x16,x24,x3,lo + ret + + +.globl _sqr_mont_384 +.private_extern _sqr_mont_384 + +.align 5 +_sqr_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for 768-bit vector + mov x4,x3 // adjust for missing b_ptr + + mov x3,x0 // save r_ptr + mov x0,sp + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + mov x1,sp + mov x0,x3 // restore r_ptr + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _sqr_n_mul_mont_383 +.private_extern _sqr_n_mul_mont_383 + +.align 5 +_sqr_n_mul_mont_383: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 768-bit vector + mov x17,x5 // save b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + mov x0,sp +Loop_sqr_383: + bl __sqr_384 + sub x2,x2,#1 // counter + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,sp + bl __mul_by_1_mont_384 + + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // just accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + cbnz x2,Loop_sqr_383 + + mov x2,x17 + ldr x17,[x17] + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.align 5 +__sqr_384: + mul x19,x12,x11 + mul x20,x13,x11 + mul x21,x14,x11 + mul x22,x15,x11 + mul x23,x16,x11 + + umulh x6,x12,x11 + umulh x7,x13,x11 + umulh x8,x14,x11 + umulh x9,x15,x11 + adds x20,x20,x6 + umulh x10,x16,x11 + adcs x21,x21,x7 + mul x7,x13,x12 + adcs x22,x22,x8 + mul x8,x14,x12 + adcs x23,x23,x9 + mul x9,x15,x12 + adc x24,xzr, x10 + mul x10,x16,x12 + + adds x21,x21,x7 + umulh x7,x13,x12 + adcs x22,x22,x8 + umulh x8,x14,x12 + adcs x23,x23,x9 + umulh x9,x15,x12 + adcs x24,x24,x10 + umulh x10,x16,x12 + adc x25,xzr,xzr + + mul x5,x11,x11 + adds x22,x22,x7 + umulh x11, x11,x11 + adcs x23,x23,x8 + mul x8,x14,x13 + adcs x24,x24,x9 + mul x9,x15,x13 + adc x25,x25,x10 + mul x10,x16,x13 + + adds x23,x23,x8 + umulh x8,x14,x13 + adcs x24,x24,x9 + umulh x9,x15,x13 + adcs x25,x25,x10 + umulh x10,x16,x13 + adc x26,xzr,xzr + + mul x6,x12,x12 + adds x24,x24,x8 + umulh x12, x12,x12 + adcs x25,x25,x9 + mul x9,x15,x14 + adc x26,x26,x10 + mul x10,x16,x14 + + adds x25,x25,x9 + umulh x9,x15,x14 + adcs x26,x26,x10 + umulh x10,x16,x14 + adc x27,xzr,xzr + mul x7,x13,x13 + adds x26,x26,x9 + umulh x13, x13,x13 + adc x27,x27,x10 + mul x8,x14,x14 + + mul x10,x16,x15 + umulh x14, x14,x14 + adds x27,x27,x10 + umulh x10,x16,x15 + mul x9,x15,x15 + adc x28,x10,xzr + + adds x19,x19,x19 + adcs x20,x20,x20 + adcs x21,x21,x21 + adcs x22,x22,x22 + adcs x23,x23,x23 + adcs x24,x24,x24 + adcs x25,x25,x25 + adcs x26,x26,x26 + umulh x15, x15,x15 + adcs x27,x27,x27 + mul x10,x16,x16 + adcs x28,x28,x28 + umulh x16, x16,x16 + adc x1,xzr,xzr + + adds x19,x19,x11 + adcs x20,x20,x6 + adcs x21,x21,x12 + adcs x22,x22,x7 + adcs x23,x23,x13 + adcs x24,x24,x8 + adcs x25,x25,x14 + stp x5,x19,[x0] + adcs x26,x26,x9 + stp x20,x21,[x0,#16] + adcs x27,x27,x15 + stp x22,x23,[x0,#32] + adcs x28,x28,x10 + stp x24,x25,[x0,#48] + adc x16,x16,x1 + stp x26,x27,[x0,#64] + stp x28,x16,[x0,#80] + + ret + +.globl _sqr_384 +.private_extern _sqr_384 + +.align 5 +_sqr_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _redc_mont_384 +.private_extern _redc_mont_384 + +.align 5 +_redc_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _from_mont_384 +.private_extern _from_mont_384 + +.align 5 +_from_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + + +.align 5 +__mul_by_1_mont_384: + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + mul x26,x4,x11 + ldp x15,x16,[x1,#32] + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + ret + + + +.align 5 +__redc_tail_mont_384: + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csel x11,x11,x19,lo + csel x12,x12,x20,lo + csel x13,x13,x21,lo + csel x14,x14,x22,lo + csel x15,x15,x23,lo + csel x16,x16,x24,lo + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + + +.globl _mul_384 +.private_extern _mul_384 + +.align 5 +_mul_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + bl __mul_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + + +.align 5 +__mul_384: + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + + umulh x5,x11,x17 + umulh x6,x12,x17 + umulh x7,x13,x17 + umulh x8,x14,x17 + umulh x9,x15,x17 + umulh x10,x16,x17 + ldr x17,[x2,8*1] + + str x19,[x0] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,xzr, x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(1+1)] + adc x25,xzr,xzr + + str x19,[x0,8*1] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(2+1)] + adc x25,xzr,xzr + + str x19,[x0,8*2] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(3+1)] + adc x25,xzr,xzr + + str x19,[x0,8*3] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(4+1)] + adc x25,xzr,xzr + + str x19,[x0,8*4] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + adc x25,xzr,xzr + + str x19,[x0,8*5] + adds x19,x20,x5 + adcs x20,x21,x6 + adcs x21,x22,x7 + adcs x22,x23,x8 + adcs x23,x24,x9 + adc x24,x25,x10 + + stp x19,x20,[x0,#48] + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ret + + +.globl _mul_382x +.private_extern _mul_382x + +.align 5 +_mul_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for two 384-bit vectors + + ldp x11,x12,[x1] + mov x26,x0 // save r_ptr + ldp x19,x20,[x1,#48] + mov x27,x1 // save a_ptr + ldp x13,x14,[x1,#16] + mov x28,x2 // save b_ptr + ldp x21,x22,[x1,#64] + ldp x15,x16,[x1,#32] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x23,x24,[x1,#80] + adcs x6,x12,x20 + ldp x11,x12,[x2] + adcs x7,x13,x21 + ldp x19,x20,[x2,#48] + adcs x8,x14,x22 + ldp x13,x14,[x2,#16] + adcs x9,x15,x23 + ldp x21,x22,[x2,#64] + adc x10,x16,x24 + ldp x15,x16,[x2,#32] + + stp x5,x6,[sp] + adds x5,x11,x19 // t1 = b->re + b->im + ldp x23,x24,[x2,#80] + adcs x6,x12,x20 + stp x7,x8,[sp,#16] + adcs x7,x13,x21 + adcs x8,x14,x22 + stp x9,x10,[sp,#32] + adcs x9,x15,x23 + stp x5,x6,[sp,#48] + adc x10,x16,x24 + stp x7,x8,[sp,#64] + stp x9,x10,[sp,#80] + + bl __mul_384 // _mul_384(ret->re, a->re, b->re) + + add x1,sp,#0 // _mul_384(ret->im, t0, t1) + add x2,sp,#48 + add x0,x26,#96 + bl __mul_384 + + add x1,x27,#48 // _mul_384(tx, a->im, b->im) + add x2,x28,#48 + add x0,sp,#0 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + add x1,x26,#96 // ret->im -= tx + add x2,sp,#0 + add x0,x26,#96 + bl __sub_mod_384x384 + + add x2,x26,#0 // ret->im -= ret->re + bl __sub_mod_384x384 + + add x1,x26,#0 // ret->re -= tx + add x2,sp,#0 + add x0,x26,#0 + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _sqr_382x +.private_extern _sqr_382x + +.align 5 +_sqr_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x19,x20,[x1,#48] + ldp x13,x14,[x1,#16] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x21,x22,[x1,#64] + adcs x6,x12,x20 + ldp x15,x16,[x1,#32] + adcs x7,x13,x21 + ldp x23,x24,[x1,#80] + adcs x8,x14,x22 + stp x5,x6,[x0] + adcs x9,x15,x23 + ldp x5,x6,[x2] + adc x10,x16,x24 + stp x7,x8,[x0,#16] + + subs x11,x11,x19 // t1 = a->re - a->im + ldp x7,x8,[x2,#16] + sbcs x12,x12,x20 + stp x9,x10,[x0,#32] + sbcs x13,x13,x21 + ldp x9,x10,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + adds x11,x11,x19 + and x21,x7,x25 + adcs x12,x12,x20 + and x22,x8,x25 + adcs x13,x13,x21 + and x23,x9,x25 + adcs x14,x14,x22 + and x24,x10,x25 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + mov x4,x1 // save a_ptr + add x1,x0,#0 // _mul_384(ret->re, t0, t1) + add x2,x0,#48 + bl __mul_384 + + add x1,x4,#0 // _mul_384(ret->im, a->re, a->im) + add x2,x4,#48 + add x0,x0,#96 + bl __mul_384 + ldr x30,[x29,#8] + + ldp x11,x12,[x0] + ldp x13,x14,[x0,#16] + adds x11,x11,x11 // add with itself + ldp x15,x16,[x0,#32] + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adcs x19,x19,x19 + adcs x20,x20,x20 + stp x11,x12,[x0] + adcs x21,x21,x21 + stp x13,x14,[x0,#16] + adcs x22,x22,x22 + stp x15,x16,[x0,#32] + adcs x23,x23,x23 + stp x19,x20,[x0,#48] + adc x24,x24,x24 + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _sqr_mont_382x +.private_extern _sqr_mont_382x + +.align 5 +_sqr_mont_382x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#112 // space for two 384-bit vectors + word + mov x4,x3 // adjust for missing b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x17,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x5,x11,x17 // t0 = a->re + a->im + adcs x6,x12,x20 + adcs x7,x13,x21 + adcs x8,x14,x22 + adcs x9,x15,x23 + adc x10,x16,x24 + + subs x19,x11,x17 // t1 = a->re - a->im + sbcs x20,x12,x20 + sbcs x21,x13,x21 + sbcs x22,x14,x22 + sbcs x23,x15,x23 + sbcs x24,x16,x24 + sbc x25,xzr,xzr // borrow flag as mask + + stp x5,x6,[sp] + stp x7,x8,[sp,#16] + stp x9,x10,[sp,#32] + stp x19,x20,[sp,#48] + stp x21,x22,[sp,#64] + stp x23,x24,[sp,#80] + str x25,[sp,#96] + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + bl __mul_mont_383_nonred // _mul_mont_384(ret->im, a->re, a->im) + + adds x19,x11,x11 // add with itself + adcs x20,x12,x12 + adcs x21,x13,x13 + adcs x22,x14,x14 + adcs x23,x15,x15 + adc x24,x16,x16 + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + ldp x11,x12,[sp] + ldr x17,[sp,#48] + ldp x13,x14,[sp,#16] + ldp x15,x16,[sp,#32] + + add x2,sp,#48 + bl __mul_mont_383_nonred // _mul_mont_384(ret->im, t0, t1) + ldr x30,[x29,#8] + + ldr x25,[sp,#96] // account for sign from a->re - a->im + ldp x19,x20,[sp] + ldp x21,x22,[sp,#16] + ldp x23,x24,[sp,#32] + + and x19,x19,x25 + and x20,x20,x25 + and x21,x21,x25 + and x22,x22,x25 + and x23,x23,x25 + and x24,x24,x25 + + subs x11,x11,x19 + sbcs x12,x12,x20 + sbcs x13,x13,x21 + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + and x21,x7,x25 + and x22,x8,x25 + and x23,x9,x25 + and x24,x10,x25 + + adds x11,x11,x19 + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#112 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + + +.align 5 +__mul_mont_383_nonred: + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + ldr x17,[x2,8*1] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*2] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*3] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*4] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*5] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + + adds x11,x20,x26 + adcs x12,x21,x27 + adcs x13,x22,x28 + adcs x14,x23,x0 + adcs x15,x24,x1 + adcs x16,x25,x3 + + ret + + +.globl _sgn0_pty_mont_384 +.private_extern _sgn0_pty_mont_384 + +.align 5 +_sgn0_pty_mont_384: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + adds x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + + +.globl _sgn0_pty_mont_384x +.private_extern _sgn0_pty_mont_384x + +.align 5 +_sgn0_pty_mont_384x: +.long 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + add x1,x1,#48 + + and x2,x11,#1 + orr x3,x11,x12 + adds x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + orr x3,x3,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x2,x2,x17 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + orr x1,x11,x12 + adds x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + orr x1,x1,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + cmp x3,#0 + csel x3,x0,x2,eq // a->re==0? prty(a->im) : prty(a->re) + + cmp x1,#0 + csel x1,x0,x2,ne // a->im!=0? sgn0(a->im) : sgn0(a->re) + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 +.long 3573752767 + ret + diff --git a/crypto/blst_src/build/mach-o/mulq_mont_256-x86_64.s b/crypto/blst_src/build/mach-o/mulq_mont_256-x86_64.s new file mode 100644 index 00000000000..d83f5440342 --- /dev/null +++ b/crypto/blst_src/build/mach-o/mulq_mont_256-x86_64.s @@ -0,0 +1,706 @@ +.text + +.globl _mul_mont_sparse_256 +.private_extern _mul_mont_sparse_256 + +.p2align 5 +_mul_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rdx),%rax + movq 0(%rsi),%r13 + movq 8(%rsi),%r14 + movq 16(%rsi),%r12 + movq 24(%rsi),%rbp + movq %rdx,%rbx + + movq %rax,%r15 + mulq %r13 + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqr_mont_sparse_256 +.private_extern _sqr_mont_sparse_256 + +.p2align 5 +_sqr_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + movq 0(%rsi),%rax + movq %rcx,%r8 + movq 8(%rsi),%r14 + movq %rdx,%rcx + movq 16(%rsi),%r12 + leaq (%rsi),%rbx + movq 24(%rsi),%rbp + + movq %rax,%r15 + mulq %rax + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + call __mulq_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulq_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + mulq %r14 + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r12 + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq 8(%rbx),%rax + adcq $0,%rdx + xorq %r14,%r14 + movq %rdx,%r13 + + movq %r9,%rdi + imulq %r8,%r9 + + + movq %rax,%r15 + mulq 0(%rsi) + addq %rax,%r10 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + xorq %r15,%r15 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r9,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rdi,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + addq %rdx,%r13 + adcq $0,%r14 + adcq $0,%r15 + movq %r10,%rdi + imulq %r8,%r10 + + + movq %rax,%r9 + mulq 0(%rsi) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + xorq %r9,%r9 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r10,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rdi,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r13 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + addq %rdx,%r14 + adcq $0,%r15 + adcq $0,%r9 + movq %r11,%rdi + imulq %r8,%r11 + + + movq %rax,%r10 + mulq 0(%rsi) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 8(%rsi) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rsi) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rsi) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r9 + xorq %r10,%r10 + + + mulq 0(%rcx) + addq %rax,%rdi + movq %r11,%rax + adcq %rdx,%rdi + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rdi,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + addq %rdx,%r15 + adcq $0,%r9 + adcq $0,%r10 + imulq %r8,%rax + movq 8(%rsp),%rsi + + + movq %rax,%r11 + mulq 0(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r12,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + movq %r14,%rbx + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rdx,%r9 + adcq $0,%r10 + + + + + movq %r15,%r12 + subq 0(%rcx),%r13 + sbbq 8(%rcx),%r14 + sbbq 16(%rcx),%r15 + movq %r9,%rbp + sbbq 24(%rcx),%r9 + sbbq $0,%r10 + + cmovcq %rax,%r13 + cmovcq %rbx,%r14 + cmovcq %r12,%r15 + movq %r13,0(%rsi) + cmovcq %rbp,%r9 + movq %r14,8(%rsi) + movq %r15,16(%rsi) + movq %r9,24(%rsi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _from_mont_256 +.private_extern _from_mont_256 + +.p2align 5 +_from_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + + + + + movq %r14,%r10 + movq %r15,%r11 + movq %r9,%r12 + + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + sbbq 24(%rbx),%r9 + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _redc_mont_256 +.private_extern _redc_mont_256 + +.p2align 5 +_redc_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_256 + + addq 32(%rsi),%r13 + adcq 40(%rsi),%r14 + movq %r13,%rax + adcq 48(%rsi),%r15 + movq %r14,%r10 + adcq 56(%rsi),%r9 + sbbq %rsi,%rsi + + + + + movq %r15,%r11 + subq 0(%rbx),%r13 + sbbq 8(%rbx),%r14 + sbbq 16(%rbx),%r15 + movq %r9,%r12 + sbbq 24(%rbx),%r9 + sbbq $0,%rsi + + cmovncq %r13,%rax + cmovncq %r14,%r10 + cmovncq %r15,%r11 + movq %rax,0(%rdi) + cmovncq %r9,%r12 + movq %r10,8(%rdi) + movq %r11,16(%rdi) + movq %r12,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulq_by_1_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + + movq %rax,%r13 + imulq %rcx,%rax + movq %rax,%r9 + + mulq 0(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq %rdx,%r13 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r10 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 16(%rbx) + movq %r10,%r14 + imulq %rcx,%r10 + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r13,%r11 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r13,%r12 + adcq $0,%rdx + movq %rdx,%r13 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r9 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/mulq_mont_384-x86_64.s b/crypto/blst_src/build/mach-o/mulq_mont_384-x86_64.s new file mode 100644 index 00000000000..0d8ac89cfc2 --- /dev/null +++ b/crypto/blst_src/build/mach-o/mulq_mont_384-x86_64.s @@ -0,0 +1,3612 @@ +.text + + + + + + + + +.p2align 5 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mul_mont_384x +.private_extern _mul_mont_384x + +.p2align 5 +_mul_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $328,%rsp +.cfi_adjust_cfa_offset 328 + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulq_384 + + + leaq 48(%rbx),%rbx + leaq 48(%rsi),%rsi + leaq 40+96(%rsp),%rdi + call __mulq_384 + + + movq 8(%rsp),%rcx + leaq -48(%rsi),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulq_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + movq %rcx,%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -328-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqr_mont_384x +.private_extern _sqr_mont_384x + +.p2align 5 +_sqr_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,8(%rsp) + movq %rsi,16(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 16(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + call __mulq_mont_384 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + movq %r14,%r12 + adcq %r9,%r9 + movq %r15,%r13 + adcq %r10,%r10 + movq %r8,%rax + adcq %r11,%r11 + movq %r9,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r10,%rbp + sbbq 16(%rcx),%r8 + sbbq 24(%rcx),%r9 + sbbq 32(%rcx),%r10 + movq %r11,%rsi + sbbq 40(%rcx),%r11 + sbbq $0,%rdx + + cmovcq %r12,%r14 + cmovcq %r13,%r15 + cmovcq %rax,%r8 + movq %r14,48(%rdi) + cmovcq %rbx,%r9 + movq %r15,56(%rdi) + cmovcq %rbp,%r10 + movq %r8,64(%rdi) + cmovcq %rsi,%r11 + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _mul_382x +.private_extern _mul_382x + +.p2align 5 +_mul_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulq_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulq_384 + + + leaq 48(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulq_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqr_382x +.private_extern _sqr_382x + +.p2align 5 +_sqr_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulq_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulq_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -8*7 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mul_384 +.private_extern _mul_384 + +.p2align 5 +_mul_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + + + movq %rdx,%rbx + call __mulq_384 + + movq 0(%rsp),%r12 +.cfi_restore %r12 + movq 8(%rsp),%rbx +.cfi_restore %rbx + movq 16(%rsp),%rbp +.cfi_restore %rbp + leaq 24(%rsp),%rsp +.cfi_adjust_cfa_offset -24 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__mulq_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rax + + movq %rax,%rbp + mulq 0(%rsi) + movq %rax,0(%rdi) + movq %rbp,%rax + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r11 + movq 8(%rbx),%rax + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,8(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,16(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,24(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,32(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%rcx + movq %rbp,%rax + adcq $0,%rdx + movq %rcx,40(%rdi) + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %rax,%r12 + movq %rax,%rax + adcq $0,%rdx + addq %r12,%r11 + adcq $0,%rdx + movq %rdx,%r12 + movq %rcx,48(%rdi) + movq %r8,56(%rdi) + movq %r9,64(%rdi) + movq %r10,72(%rdi) + movq %r11,80(%rdi) + movq %r12,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqr_384 +.private_extern _sqr_384 + +.p2align 5 +_sqr_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + call __sqrq_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__sqrq_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r15 + movq 16(%rsi),%rcx + movq 24(%rsi),%rbx + + + movq %rax,%r14 + mulq %r15 + movq %rax,%r9 + movq %r14,%rax + movq 32(%rsi),%rbp + movq %rdx,%r10 + + mulq %rcx + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + movq 40(%rsi),%rsi + movq %rdx,%r11 + + mulq %rbx + addq %rax,%r11 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq %rbp + addq %rax,%r12 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq %rsi + addq %rax,%r13 + movq %r14,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq %rax + xorq %r8,%r8 + movq %rax,0(%rdi) + movq %r15,%rax + addq %r9,%r9 + adcq $0,%r8 + addq %rdx,%r9 + adcq $0,%r8 + movq %r9,8(%rdi) + + mulq %rcx + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbx + addq %rax,%r12 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rbp + addq %rax,%r13 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq %rsi + addq %rax,%r14 + movq %r15,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq %rax + xorq %r9,%r9 + addq %rax,%r8 + movq %rcx,%rax + addq %r10,%r10 + adcq %r11,%r11 + adcq $0,%r9 + addq %r8,%r10 + adcq %rdx,%r11 + adcq $0,%r9 + movq %r10,16(%rdi) + + mulq %rbx + addq %rax,%r13 + movq %rcx,%rax + adcq $0,%rdx + movq %r11,24(%rdi) + movq %rdx,%r8 + + mulq %rbp + addq %rax,%r14 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq %rsi + addq %rax,%r15 + movq %rcx,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + xorq %r11,%r11 + addq %rax,%r9 + movq %rbx,%rax + addq %r12,%r12 + adcq %r13,%r13 + adcq $0,%r11 + addq %r9,%r12 + adcq %rdx,%r13 + adcq $0,%r11 + movq %r12,32(%rdi) + + + mulq %rbp + addq %rax,%r15 + movq %rbx,%rax + adcq $0,%rdx + movq %r13,40(%rdi) + movq %rdx,%r8 + + mulq %rsi + addq %rax,%rcx + movq %rbx,%rax + adcq $0,%rdx + addq %r8,%rcx + adcq $0,%rdx + movq %rdx,%rbx + + mulq %rax + xorq %r12,%r12 + addq %rax,%r11 + movq %rbp,%rax + addq %r14,%r14 + adcq %r15,%r15 + adcq $0,%r12 + addq %r11,%r14 + adcq %rdx,%r15 + movq %r14,48(%rdi) + adcq $0,%r12 + movq %r15,56(%rdi) + + + mulq %rsi + addq %rax,%rbx + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq %rax + xorq %r13,%r13 + addq %rax,%r12 + movq %rsi,%rax + addq %rcx,%rcx + adcq %rbx,%rbx + adcq $0,%r13 + addq %r12,%rcx + adcq %rdx,%rbx + movq %rcx,64(%rdi) + adcq $0,%r13 + movq %rbx,72(%rdi) + + + mulq %rax + addq %r13,%rax + addq %rbp,%rbp + adcq $0,%rdx + addq %rbp,%rax + adcq $0,%rdx + movq %rax,80(%rdi) + movq %rdx,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqr_mont_384 +.private_extern _sqr_mont_384 + +.p2align 5 +_sqr_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $120,%rsp +.cfi_adjust_cfa_offset 8*15 + + + movq %rcx,96(%rsp) + movq %rdx,104(%rsp) + movq %rdi,112(%rsp) + + movq %rsp,%rdi + call __sqrq_384 + + leaq 0(%rsp),%rsi + movq 96(%rsp),%rcx + movq 104(%rsp),%rbx + movq 112(%rsp),%rdi + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 120(%rsp),%r8 + movq 120(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*21 + + .byte 0xf3,0xc3 +.cfi_endproc + + + + +.globl _redc_mont_384 +.private_extern _redc_mont_384 + +.p2align 5 +_redc_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + + + +.globl _from_mont_384 +.private_extern _from_mont_384 + +.p2align 5 +_from_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulq_by_1_mont_384 + + + + + + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulq_by_1_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r8 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r8,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r9 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r10 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %r9,%r15 + imulq %rcx,%r9 + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 32(%rbx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 40(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r9,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %r10,%r8 + imulq %rcx,%r10 + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rbx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r8 + movq %r10,%rax + adcq %rdx,%r8 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rbx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %r11,%r9 + imulq %rcx,%r11 + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r8,%r15 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rbx) + addq %rax,%r9 + movq %r11,%rax + adcq %rdx,%r9 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %r12,%r10 + imulq %rcx,%r12 + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r9,%r8 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %r13,%r11 + imulq %rcx,%r13 + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rbx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r9 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rbx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__redc_tail_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sgn0_pty_mont_384 +.private_extern _sgn0_pty_mont_384 + +.p2align 5 +_sgn0_pty_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sgn0_pty_mont_384x +.private_extern _sgn0_pty_mont_384x + +.p2align 5 +_sgn0_pty_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulq_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mul_mont_384 +.private_extern _mul_mont_384 + +.p2align 5 +_mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $24,%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq 0(%rdx),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + movq %rdx,%rbx + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + + call __mulq_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -72 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulq_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rdi + mulq %r14 + movq %rax,%r8 + movq %rdi,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%rbp + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + xorq %r15,%r15 + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r8,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq %rdx,%r14 + adcq $0,%r15 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r9,%rbp + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r14 + adcq $0,%rdx + xorq %r8,%r8 + addq %rax,%r14 + movq %r9,%rax + adcq %rdx,%r15 + adcq $0,%r8 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r9,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq %rdx,%r15 + adcq $0,%r8 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r10 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r10,%rbp + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r15 + adcq $0,%rdx + xorq %r9,%r9 + addq %rax,%r15 + movq %r10,%rax + adcq %rdx,%r8 + adcq $0,%r9 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r10,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r11 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq %rdx,%r8 + adcq $0,%r9 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r11 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r11,%rbp + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r8 + adcq $0,%rdx + xorq %r10,%r10 + addq %rax,%r8 + movq %r11,%rax + adcq %rdx,%r9 + adcq $0,%r10 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r11,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r12 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq %rdx,%r9 + adcq $0,%r10 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r12 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r12,%rbp + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r9 + adcq $0,%rdx + xorq %r11,%r11 + addq %rax,%r9 + movq %r12,%rax + adcq %rdx,%r10 + adcq $0,%r11 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r12,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r13 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %rbp,%r8 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq %rdx,%r10 + adcq $0,%r11 + + movq %rax,%rdi + mulq 0(%rsi) + addq %rax,%r13 + movq %rdi,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + movq %r13,%rbp + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r8 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rdi,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rsi) + addq %r12,%r10 + adcq $0,%rdx + xorq %r12,%r12 + addq %rax,%r10 + movq %r13,%rax + adcq %rdx,%r11 + adcq $0,%r12 + + mulq 0(%rcx) + addq %rax,%rbp + movq %r13,%rax + adcq %rdx,%rbp + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r14 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r15 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 24(%rcx) + addq %rbp,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %rbp,%r9 + adcq $0,%rdx + movq %rdx,%rbp + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %rbp,%r10 + adcq %rdx,%r11 + adcq $0,%r12 + + + + + movq 16(%rsp),%rdi + subq 0(%rcx),%r14 + movq %r15,%rdx + sbbq 8(%rcx),%r15 + movq %r8,%rbx + sbbq 16(%rcx),%r8 + movq %r9,%rsi + sbbq 24(%rcx),%r9 + movq %r10,%rbp + sbbq 32(%rcx),%r10 + movq %r11,%r13 + sbbq 40(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rdx,%r15 + cmovcq %rbx,%r8 + movq %r14,0(%rdi) + cmovcq %rsi,%r9 + movq %r15,8(%rdi) + cmovcq %rbp,%r10 + movq %r8,16(%rdi) + cmovcq %r13,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqr_n_mul_mont_384 +.private_extern _sqr_n_mul_mont_384 + +.p2align 5 +_sqr_n_mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 8*17 + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +L$oop_sqr_384: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movd %xmm1,%edx + leaq 0(%rdi),%rsi + decl %edx + jnz L$oop_sqr_384 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*23 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqr_n_mul_mont_383 +.private_extern _sqr_n_mul_mont_383 + +.p2align 5 +_sqr_n_mul_mont_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 8*17 + + + movq %r8,0(%rsp) + movq %rdi,8(%rsp) + movq %rcx,16(%rsp) + leaq 32(%rsp),%rdi + movq %r9,24(%rsp) + movq (%r9),%xmm2 + +L$oop_sqr_383: + movd %edx,%xmm1 + + call __sqrq_384 + + leaq 0(%rdi),%rsi + movq 0(%rsp),%rcx + movq 16(%rsp),%rbx + call __mulq_by_1_mont_384 + + movd %xmm1,%edx + addq 48(%rsi),%r14 + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + leaq 0(%rdi),%rsi + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + decl %edx + jnz L$oop_sqr_383 + +.byte 102,72,15,126,208 + movq %rbx,%rcx + movq 24(%rsp),%rbx + + + + + + + movq %r8,%r12 + movq %r9,%r13 + + call __mulq_mont_384 + + leaq 136(%rsp),%r8 + movq 136(%rsp),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -8*23 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulq_mont_383_nonred: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq %rax,%rbp + mulq %r14 + movq %rax,%r8 + movq %rbp,%rax + movq %rdx,%r9 + + mulq %r15 + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r12 + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + movq %r8,%r15 + imulq 8(%rsp),%r8 + + mulq %r13 + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r13 + + mulq 40(%rsi) + addq %rax,%r13 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rcx) + addq %rax,%r15 + movq %r8,%rax + adcq %rdx,%r15 + + mulq 8(%rcx) + addq %rax,%r9 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r9 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rcx) + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rcx) + addq %r15,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rcx) + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rcx) + addq %rax,%r13 + movq 8(%rbx),%rax + adcq $0,%rdx + addq %r15,%r13 + adcq %rdx,%r14 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 8(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r10 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r11 + adcq $0,%rdx + movq %rdx,%r15 + + movq %r9,%r8 + imulq 8(%rsp),%r9 + + mulq 24(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 32(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 40(%rsi) + addq %r15,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rcx) + addq %rax,%r8 + movq %r9,%rax + adcq %rdx,%r8 + + mulq 8(%rcx) + addq %rax,%r10 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r10 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rcx) + addq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 24(%rcx) + addq %r8,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rcx) + addq %rax,%r13 + movq %r9,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rcx) + addq %rax,%r14 + movq 16(%rbx),%rax + adcq $0,%rdx + addq %r8,%r14 + adcq %rdx,%r15 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 8(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r11 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 16(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r12 + adcq $0,%rdx + movq %rdx,%r8 + + movq %r10,%r9 + imulq 8(%rsp),%r10 + + mulq 24(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r13 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 32(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r8,%r14 + adcq $0,%rdx + movq %rdx,%r8 + + mulq 40(%rsi) + addq %r8,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r8 + + mulq 0(%rcx) + addq %rax,%r9 + movq %r10,%rax + adcq %rdx,%r9 + + mulq 8(%rcx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r11 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rcx) + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 24(%rcx) + addq %r9,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rcx) + addq %rax,%r14 + movq %r10,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rcx) + addq %rax,%r15 + movq 24(%rbx),%rax + adcq $0,%rdx + addq %r9,%r15 + adcq %rdx,%r8 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 8(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r12 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 16(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r13 + adcq $0,%rdx + movq %rdx,%r9 + + movq %r11,%r10 + imulq 8(%rsp),%r11 + + mulq 24(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r14 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 32(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r9,%r15 + adcq $0,%rdx + movq %rdx,%r9 + + mulq 40(%rsi) + addq %r9,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r9 + + mulq 0(%rcx) + addq %rax,%r10 + movq %r11,%rax + adcq %rdx,%r10 + + mulq 8(%rcx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r12 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rcx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rcx) + addq %r10,%r14 + adcq $0,%rdx + addq %rax,%r14 + movq %r11,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rcx) + addq %rax,%r15 + movq %r11,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rcx) + addq %rax,%r8 + movq 32(%rbx),%rax + adcq $0,%rdx + addq %r10,%r8 + adcq %rdx,%r9 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 8(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + movq %r12,%r11 + imulq 8(%rsp),%r12 + + mulq 24(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 32(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r10,%r8 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 40(%rsi) + addq %r10,%r9 + adcq $0,%rdx + addq %rax,%r9 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rcx) + addq %rax,%r11 + movq %r12,%rax + adcq %rdx,%r11 + + mulq 8(%rcx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r13 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rcx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rcx) + addq %r11,%r15 + adcq $0,%rdx + addq %rax,%r15 + movq %r12,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rcx) + addq %rax,%r8 + movq %r12,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rcx) + addq %rax,%r9 + movq 40(%rbx),%rax + adcq $0,%rdx + addq %r11,%r9 + adcq %rdx,%r10 + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 8(%rsi) + addq %rax,%r14 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rsi) + addq %rax,%r15 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + movq %r13,%r12 + imulq 8(%rsp),%r13 + + mulq 24(%rsi) + addq %rax,%r8 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r8 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 32(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + addq %r11,%r9 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 40(%rsi) + addq %r11,%r10 + adcq $0,%rdx + addq %rax,%r10 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq 0(%rcx) + addq %rax,%r12 + movq %r13,%rax + adcq %rdx,%r12 + + mulq 8(%rcx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r14 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 16(%rcx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r15 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 24(%rcx) + addq %r12,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r13,%rax + adcq $0,%rdx + movq %rdx,%r12 + + mulq 32(%rcx) + addq %rax,%r9 + movq %r13,%rax + adcq $0,%rdx + addq %r12,%r9 + adcq $0,%rdx + movq %rdx,%r12 + + mulq 40(%rcx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r12,%r10 + adcq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqr_mont_382x +.private_extern _sqr_mont_382x + +.p2align 5 +_sqr_mont_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rsi,16(%rsp) + movq %rdi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rax + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq 24(%rsp),%rdi + call __mulq_mont_383_nonred + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %r8,64(%rdi) + movq %r9,72(%rdi) + movq %r10,80(%rdi) + movq %r11,88(%rdi) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rax + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%r12 + movq 32+24(%rsp),%r13 + + call __mulq_mont_383_nonred + movq 32+96(%rsp),%rsi + movq 32+0(%rsp),%r12 + movq 32+8(%rsp),%r13 + andq %rsi,%r12 + movq 32+16(%rsp),%rax + andq %rsi,%r13 + movq 32+24(%rsp),%rbx + andq %rsi,%rax + movq 32+32(%rsp),%rbp + andq %rsi,%rbx + andq %rsi,%rbp + andq 32+40(%rsp),%rsi + + subq %r12,%r14 + movq 0(%rcx),%r12 + sbbq %r13,%r15 + movq 8(%rcx),%r13 + sbbq %rax,%r8 + movq 16(%rcx),%rax + sbbq %rbx,%r9 + movq 24(%rcx),%rbx + sbbq %rbp,%r10 + movq 32(%rcx),%rbp + sbbq %rsi,%r11 + sbbq %rsi,%rsi + + andq %rsi,%r12 + andq %rsi,%r13 + andq %rsi,%rax + andq %rsi,%rbx + andq %rsi,%rbp + andq 40(%rcx),%rsi + + addq %r12,%r14 + adcq %r13,%r15 + adcq %rax,%r8 + adcq %rbx,%r9 + adcq %rbp,%r10 + adcq %rsi,%r11 + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/mulx_mont_256-x86_64.s b/crypto/blst_src/build/mach-o/mulx_mont_256-x86_64.s new file mode 100644 index 00000000000..178372f41b2 --- /dev/null +++ b/crypto/blst_src/build/mach-o/mulx_mont_256-x86_64.s @@ -0,0 +1,619 @@ +.text + +.globl _mulx_mont_sparse_256 +.private_extern _mulx_mont_sparse_256 + +.p2align 5 +_mulx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqrx_mont_sparse_256 +.private_extern _sqrx_mont_sparse_256 + +.p2align 5 +_sqrx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + movq %rcx,%r8 + movq %rdx,%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rbp + movq 24(%rsi),%r9 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%rax,%r11 + call __mulx_mont_sparse_256 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulx_mont_sparse_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + mulxq %r15,%r15,%r12 + mulxq %rbp,%rbp,%r13 + addq %r15,%r11 + mulxq %r9,%r9,%r14 + movq 8(%rbx),%rdx + adcq %rbp,%r12 + adcq %r9,%r13 + adcq $0,%r14 + + movq %rax,%r10 + imulq %r8,%rax + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r11 + adcxq %r9,%r12 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r14 + adcxq %r15,%r9 + adoxq %r9,%r15 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r10 + adoxq %r11,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r12 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r12 + adoxq %r9,%r13 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 16(%rbx),%rdx + adcxq %rbp,%r13 + adoxq %r9,%r14 + adcxq %r10,%r14 + adoxq %r10,%r15 + adcxq %r10,%r15 + adoxq %r10,%r10 + adcq $0,%r10 + movq %rax,%r11 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r12 + adcxq %r9,%r13 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r15 + adcxq %r10,%r9 + adoxq %r9,%r10 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r11 + adoxq %r12,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r13 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r13 + adoxq %r9,%r14 + + mulxq 24+128(%rcx),%rbp,%r9 + movq 24(%rbx),%rdx + adcxq %rbp,%r14 + adoxq %r9,%r15 + adcxq %r11,%r15 + adoxq %r11,%r10 + adcxq %r11,%r10 + adoxq %r11,%r11 + adcq $0,%r11 + movq %rax,%r12 + imulq %r8,%rax + + + xorq %rbp,%rbp + mulxq 0+128(%rsi),%rbp,%r9 + adoxq %rbp,%r13 + adcxq %r9,%r14 + + mulxq 8+128(%rsi),%rbp,%r9 + adoxq %rbp,%r14 + adcxq %r9,%r15 + + mulxq 16+128(%rsi),%rbp,%r9 + adoxq %rbp,%r15 + adcxq %r9,%r10 + + mulxq 24+128(%rsi),%rbp,%r9 + movq %rax,%rdx + adoxq %rbp,%r10 + adcxq %r11,%r9 + adoxq %r9,%r11 + + + mulxq 0+128(%rcx),%rbp,%rax + adcxq %rbp,%r12 + adoxq %r13,%rax + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%rax + adoxq %r9,%r14 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %rax,%rdx + adcxq %rbp,%r15 + adoxq %r9,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + adoxq %r12,%r12 + adcq $0,%r12 + imulq %r8,%rdx + + + xorq %rbp,%rbp + mulxq 0+128(%rcx),%r13,%r9 + adcxq %rax,%r13 + adoxq %r9,%r14 + + mulxq 8+128(%rcx),%rbp,%r9 + adcxq %rbp,%r14 + adoxq %r9,%r15 + + mulxq 16+128(%rcx),%rbp,%r9 + adcxq %rbp,%r15 + adoxq %r9,%r10 + + mulxq 24+128(%rcx),%rbp,%r9 + movq %r14,%rdx + leaq 128(%rcx),%rcx + adcxq %rbp,%r10 + adoxq %r9,%r11 + movq %r15,%rax + adcxq %r13,%r11 + adoxq %r13,%r12 + adcq $0,%r12 + + + + + movq %r10,%rbp + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + sbbq 16(%rcx),%r10 + movq %r11,%r9 + sbbq 24(%rcx),%r11 + sbbq $0,%r12 + + cmovcq %rdx,%r14 + cmovcq %rax,%r15 + cmovcq %rbp,%r10 + movq %r14,0(%rdi) + cmovcq %r9,%r11 + movq %r15,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _fromx_mont_256 +.private_extern _fromx_mont_256 + +.p2align 5 +_fromx_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + + + + + movq %r15,%rdx + movq %r10,%r12 + movq %r11,%r13 + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + sbbq 24(%rbx),%r11 + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _redcx_mont_256 +.private_extern _redcx_mont_256 + +.p2align 5 +_redcx_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_256 + + addq 32(%rsi),%r14 + adcq 40(%rsi),%r15 + movq %r14,%rax + adcq 48(%rsi),%r10 + movq %r15,%rdx + adcq 56(%rsi),%r11 + sbbq %rsi,%rsi + + + + + movq %r10,%r12 + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r10 + movq %r11,%r13 + sbbq 24(%rbx),%r11 + sbbq $0,%rsi + + cmovncq %r14,%rax + cmovncq %r15,%rdx + cmovncq %r10,%r12 + movq %rax,0(%rdi) + cmovncq %r11,%r13 + movq %rdx,8(%rdi) + movq %r12,16(%rdi) + movq %r13,24(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulx_by_1_mont_256: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rax + movq 8(%rsi),%r11 + movq 16(%rsi),%r12 + movq 24(%rsi),%r13 + + movq %rax,%r14 + imulq %rcx,%rax + movq %rax,%r10 + + mulq 0(%rbx) + addq %rax,%r14 + movq %r10,%rax + adcq %rdx,%r14 + + mulq 8(%rbx) + addq %rax,%r11 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r11 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 16(%rbx) + movq %r11,%r15 + imulq %rcx,%r11 + addq %rax,%r12 + movq %r10,%rax + adcq $0,%rdx + addq %r14,%r12 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 24(%rbx) + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r14,%r13 + adcq $0,%rdx + movq %rdx,%r14 + + mulq 0(%rbx) + addq %rax,%r15 + movq %r11,%rax + adcq %rdx,%r15 + + mulq 8(%rbx) + addq %rax,%r12 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r12 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 16(%rbx) + movq %r12,%r10 + imulq %rcx,%r12 + addq %rax,%r13 + movq %r11,%rax + adcq $0,%rdx + addq %r15,%r13 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 24(%rbx) + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r15,%r14 + adcq $0,%rdx + movq %rdx,%r15 + + mulq 0(%rbx) + addq %rax,%r10 + movq %r12,%rax + adcq %rdx,%r10 + + mulq 8(%rbx) + addq %rax,%r13 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r13 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 16(%rbx) + movq %r13,%r11 + imulq %rcx,%r13 + addq %rax,%r14 + movq %r12,%rax + adcq $0,%rdx + addq %r10,%r14 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 24(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r10,%r15 + adcq $0,%rdx + movq %rdx,%r10 + + mulq 0(%rbx) + addq %rax,%r11 + movq %r13,%rax + adcq %rdx,%r11 + + mulq 8(%rbx) + addq %rax,%r14 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r14 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 16(%rbx) + addq %rax,%r15 + movq %r13,%rax + adcq $0,%rdx + addq %r11,%r15 + adcq $0,%rdx + movq %rdx,%r11 + + mulq 24(%rbx) + addq %rax,%r10 + movq %r14,%rax + adcq $0,%rdx + addq %r11,%r10 + adcq $0,%rdx + movq %rdx,%r11 + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/mulx_mont_384-x86_64.s b/crypto/blst_src/build/mach-o/mulx_mont_384-x86_64.s new file mode 100644 index 00000000000..95d3dadcc67 --- /dev/null +++ b/crypto/blst_src/build/mach-o/mulx_mont_384-x86_64.s @@ -0,0 +1,2960 @@ +.text + + + + + + + + +.p2align 5 +__sub_mod_384x384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + movq 48(%rsi),%r14 + + subq 0(%rdx),%r8 + movq 56(%rsi),%r15 + sbbq 8(%rdx),%r9 + movq 64(%rsi),%rax + sbbq 16(%rdx),%r10 + movq 72(%rsi),%rbx + sbbq 24(%rdx),%r11 + movq 80(%rsi),%rbp + sbbq 32(%rdx),%r12 + movq 88(%rsi),%rsi + sbbq 40(%rdx),%r13 + movq %r8,0(%rdi) + sbbq 48(%rdx),%r14 + movq 0(%rcx),%r8 + movq %r9,8(%rdi) + sbbq 56(%rdx),%r15 + movq 8(%rcx),%r9 + movq %r10,16(%rdi) + sbbq 64(%rdx),%rax + movq 16(%rcx),%r10 + movq %r11,24(%rdi) + sbbq 72(%rdx),%rbx + movq 24(%rcx),%r11 + movq %r12,32(%rdi) + sbbq 80(%rdx),%rbp + movq 32(%rcx),%r12 + movq %r13,40(%rdi) + sbbq 88(%rdx),%rsi + movq 40(%rcx),%r13 + sbbq %rdx,%rdx + + andq %rdx,%r8 + andq %rdx,%r9 + andq %rdx,%r10 + andq %rdx,%r11 + andq %rdx,%r12 + andq %rdx,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + movq %r14,48(%rdi) + adcq %r10,%rax + movq %r15,56(%rdi) + adcq %r11,%rbx + movq %rax,64(%rdi) + adcq %r12,%rbp + movq %rbx,72(%rdi) + adcq %r13,%rsi + movq %rbp,80(%rdi) + movq %rsi,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__add_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 0(%rdx),%r8 + adcq 8(%rdx),%r9 + adcq 16(%rdx),%r10 + movq %r8,%r14 + adcq 24(%rdx),%r11 + movq %r9,%r15 + adcq 32(%rdx),%r12 + movq %r10,%rax + adcq 40(%rdx),%r13 + movq %r11,%rbx + sbbq %rdx,%rdx + + subq 0(%rcx),%r8 + sbbq 8(%rcx),%r9 + movq %r12,%rbp + sbbq 16(%rcx),%r10 + sbbq 24(%rcx),%r11 + sbbq 32(%rcx),%r12 + movq %r13,%rsi + sbbq 40(%rcx),%r13 + sbbq $0,%rdx + + cmovcq %r14,%r8 + cmovcq %r15,%r9 + cmovcq %rax,%r10 + movq %r8,0(%rdi) + cmovcq %rbx,%r11 + movq %r9,8(%rdi) + cmovcq %rbp,%r12 + movq %r10,16(%rdi) + cmovcq %rsi,%r13 + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__sub_mod_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + +__sub_mod_384_a_is_loaded: + subq 0(%rdx),%r8 + movq 0(%rcx),%r14 + sbbq 8(%rdx),%r9 + movq 8(%rcx),%r15 + sbbq 16(%rdx),%r10 + movq 16(%rcx),%rax + sbbq 24(%rdx),%r11 + movq 24(%rcx),%rbx + sbbq 32(%rdx),%r12 + movq 32(%rcx),%rbp + sbbq 40(%rdx),%r13 + movq 40(%rcx),%rsi + sbbq %rdx,%rdx + + andq %rdx,%r14 + andq %rdx,%r15 + andq %rdx,%rax + andq %rdx,%rbx + andq %rdx,%rbp + andq %rdx,%rsi + + addq %r14,%r8 + adcq %r15,%r9 + movq %r8,0(%rdi) + adcq %rax,%r10 + movq %r9,8(%rdi) + adcq %rbx,%r11 + movq %r10,16(%rdi) + adcq %rbp,%r12 + movq %r11,24(%rdi) + adcq %rsi,%r13 + movq %r12,32(%rdi) + movq %r13,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mulx_mont_384x +.private_extern _mulx_mont_384x + +.p2align 5 +_mulx_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $328,%rsp +.cfi_adjust_cfa_offset 328 + + + movq %rdx,%rbx + movq %rdi,32(%rsp) + movq %rsi,24(%rsp) + movq %rdx,16(%rsp) + movq %rcx,8(%rsp) + movq %r8,0(%rsp) + + + + + leaq 40(%rsp),%rdi + call __mulx_384 + + + leaq 48(%rbx),%rbx + leaq 128+48(%rsi),%rsi + leaq 96(%rdi),%rdi + call __mulx_384 + + + movq 8(%rsp),%rcx + leaq (%rbx),%rsi + leaq -48(%rbx),%rdx + leaq 40+192+48(%rsp),%rdi + call __add_mod_384 + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq -48(%rdi),%rdi + call __add_mod_384 + + leaq (%rdi),%rbx + leaq 48(%rdi),%rsi + call __mulx_384 + + + leaq (%rdi),%rsi + leaq 40(%rsp),%rdx + movq 8(%rsp),%rcx + call __sub_mod_384x384 + + leaq (%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq 40(%rsp),%rsi + leaq 40+96(%rsp),%rdx + leaq 40(%rsp),%rdi + call __sub_mod_384x384 + + leaq (%rcx),%rbx + + + leaq 40(%rsp),%rsi + movq 0(%rsp),%rcx + movq 32(%rsp),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + + leaq 40+192(%rsp),%rsi + movq 0(%rsp),%rcx + leaq 48(%rdi),%rdi + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + leaq 328(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -328-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqrx_mont_384x +.private_extern _sqrx_mont_384x + +.p2align 5 +_sqrx_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + leaq 48(%rsi),%rdx + leaq 32(%rsp),%rdi + call __add_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rdx + leaq 32+48(%rsp),%rdi + call __sub_mod_384 + + + movq 24(%rsp),%rsi + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + movq %rdx,%r8 + adcq %r12,%r12 + movq %r15,%r9 + adcq %rdi,%rdi + movq %rax,%r10 + adcq %rbp,%rbp + movq %r12,%r11 + sbbq %rsi,%rsi + + subq 0(%rcx),%rdx + sbbq 8(%rcx),%r15 + movq %rdi,%r13 + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r12 + sbbq 32(%rcx),%rdi + movq %rbp,%r14 + sbbq 40(%rcx),%rbp + sbbq $0,%rsi + + cmovcq %r8,%rdx + cmovcq %r9,%r15 + cmovcq %r10,%rax + movq %rdx,48(%rbx) + cmovcq %r11,%r12 + movq %r15,56(%rbx) + cmovcq %r13,%rdi + movq %rax,64(%rbx) + cmovcq %r14,%rbp + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _mulx_382x +.private_extern _mulx_382x + +.p2align 5 +_mulx_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + leaq 96(%rdi),%rdi + movq %rsi,0(%rsp) + movq %rdx,8(%rsp) + movq %rdi,16(%rsp) + movq %rcx,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + addq 48(%rsi),%r8 + adcq 56(%rsi),%r9 + adcq 64(%rsi),%r10 + adcq 72(%rsi),%r11 + adcq 80(%rsi),%r12 + adcq 88(%rsi),%r13 + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + + movq 0(%rdx),%r8 + movq 8(%rdx),%r9 + movq 16(%rdx),%r10 + movq 24(%rdx),%r11 + movq 32(%rdx),%r12 + movq 40(%rdx),%r13 + + addq 48(%rdx),%r8 + adcq 56(%rdx),%r9 + adcq 64(%rdx),%r10 + adcq 72(%rdx),%r11 + adcq 80(%rdx),%r12 + adcq 88(%rdx),%r13 + + movq %r8,32+48(%rsp) + movq %r9,32+56(%rsp) + movq %r10,32+64(%rsp) + movq %r11,32+72(%rsp) + movq %r12,32+80(%rsp) + movq %r13,32+88(%rsp) + + + leaq 32+0(%rsp),%rsi + leaq 32+48(%rsp),%rbx + call __mulx_384 + + + movq 0(%rsp),%rsi + movq 8(%rsp),%rbx + leaq -96(%rdi),%rdi + call __mulx_384 + + + leaq 48+128(%rsi),%rsi + leaq 48(%rbx),%rbx + leaq 32(%rsp),%rdi + call __mulx_384 + + + movq 16(%rsp),%rsi + leaq 32(%rsp),%rdx + movq 24(%rsp),%rcx + movq %rsi,%rdi + call __sub_mod_384x384 + + + leaq 0(%rdi),%rsi + leaq -96(%rdi),%rdx + call __sub_mod_384x384 + + + leaq -96(%rdi),%rsi + leaq 32(%rsp),%rdx + leaq -96(%rdi),%rdi + call __sub_mod_384x384 + + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqrx_382x +.private_extern _sqrx_382x + +.p2align 5 +_sqrx_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rsi +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rcx + + + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%rbx + movq 32(%rsi),%rbp + movq 40(%rsi),%rdx + + movq %r14,%r8 + addq 48(%rsi),%r14 + movq %r15,%r9 + adcq 56(%rsi),%r15 + movq %rax,%r10 + adcq 64(%rsi),%rax + movq %rbx,%r11 + adcq 72(%rsi),%rbx + movq %rbp,%r12 + adcq 80(%rsi),%rbp + movq %rdx,%r13 + adcq 88(%rsi),%rdx + + movq %r14,0(%rdi) + movq %r15,8(%rdi) + movq %rax,16(%rdi) + movq %rbx,24(%rdi) + movq %rbp,32(%rdi) + movq %rdx,40(%rdi) + + + leaq 48(%rsi),%rdx + leaq 48(%rdi),%rdi + call __sub_mod_384_a_is_loaded + + + leaq (%rdi),%rsi + leaq -48(%rdi),%rbx + leaq -48(%rdi),%rdi + call __mulx_384 + + + movq (%rsp),%rsi + leaq 48(%rsi),%rbx + leaq 96(%rdi),%rdi + call __mulx_384 + + movq 0(%rdi),%r8 + movq 8(%rdi),%r9 + movq 16(%rdi),%r10 + movq 24(%rdi),%r11 + movq 32(%rdi),%r12 + movq 40(%rdi),%r13 + movq 48(%rdi),%r14 + movq 56(%rdi),%r15 + movq 64(%rdi),%rax + movq 72(%rdi),%rbx + movq 80(%rdi),%rbp + addq %r8,%r8 + movq 88(%rdi),%rdx + adcq %r9,%r9 + movq %r8,0(%rdi) + adcq %r10,%r10 + movq %r9,8(%rdi) + adcq %r11,%r11 + movq %r10,16(%rdi) + adcq %r12,%r12 + movq %r11,24(%rdi) + adcq %r13,%r13 + movq %r12,32(%rdi) + adcq %r14,%r14 + movq %r13,40(%rdi) + adcq %r15,%r15 + movq %r14,48(%rdi) + adcq %rax,%rax + movq %r15,56(%rdi) + adcq %rbx,%rbx + movq %rax,64(%rdi) + adcq %rbp,%rbp + movq %rbx,72(%rdi) + adcq %rdx,%rdx + movq %rbp,80(%rdi) + movq %rdx,88(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -8*7 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mulx_384 +.private_extern _mulx_384 + +.p2align 5 +_mulx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + + + movq %rdx,%rbx + call __mulx_384 + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 + + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__mulx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rbx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + leaq -128(%rsi),%rsi + + mulxq %r14,%r9,%rcx + xorq %rbp,%rbp + + mulxq %r15,%r8,%rax + adcxq %rcx,%r8 + movq %r9,0(%rdi) + + mulxq %r10,%r9,%rcx + adcxq %rax,%r9 + + mulxq %r11,%r10,%rax + adcxq %rcx,%r10 + + mulxq %r12,%r11,%rcx + adcxq %rax,%r11 + + mulxq %r13,%r12,%r13 + movq 8(%rbx),%rdx + adcxq %rcx,%r12 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,8(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 16(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,16(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 24(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,24(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 32(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,32(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq 40(%rbx),%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + mulxq %r14,%rax,%rcx + adcxq %r8,%rax + adoxq %rcx,%r9 + movq %rax,40(%rdi) + + mulxq %r15,%r8,%rcx + adcxq %r9,%r8 + adoxq %rcx,%r10 + + mulxq 128+16(%rsi),%r9,%rax + adcxq %r10,%r9 + adoxq %rax,%r11 + + mulxq 128+24(%rsi),%r10,%rcx + adcxq %r11,%r10 + adoxq %rcx,%r12 + + mulxq 128+32(%rsi),%r11,%rax + adcxq %r12,%r11 + adoxq %r13,%rax + + mulxq 128+40(%rsi),%r12,%r13 + movq %rax,%rdx + adcxq %rax,%r12 + adoxq %rbp,%r13 + adcxq %rbp,%r13 + movq %r8,48(%rdi) + movq %r9,56(%rdi) + movq %r10,64(%rdi) + movq %r11,72(%rdi) + movq %r12,80(%rdi) + movq %r13,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqrx_384 +.private_extern _sqrx_384 + +.p2align 5 +_sqrx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + pushq %rdi +.cfi_adjust_cfa_offset 8 + + + call __sqrx_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__sqrx_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%rdx + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%rcx + movq 32(%rsi),%rbx + + + mulxq %r14,%r8,%rdi + movq 40(%rsi),%rbp + mulxq %r15,%r9,%rax + addq %rdi,%r9 + mulxq %rcx,%r10,%rdi + adcq %rax,%r10 + mulxq %rbx,%r11,%rax + adcq %rdi,%r11 + mulxq %rbp,%r12,%r13 + movq %r14,%rdx + adcq %rax,%r12 + adcq $0,%r13 + + + xorq %r14,%r14 + mulxq %r15,%rdi,%rax + adcxq %rdi,%r10 + adoxq %rax,%r11 + + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r11 + adoxq %rax,%r12 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbp,%rdi,%rax + movq %r15,%rdx + adcxq %rdi,%r13 + adoxq %r14,%rax + adcxq %rax,%r14 + + + xorq %r15,%r15 + mulxq %rcx,%rdi,%rax + adcxq %rdi,%r12 + adoxq %rax,%r13 + + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r13 + adoxq %rax,%r14 + + mulxq %rbp,%rdi,%rax + movq %rcx,%rdx + adcxq %rdi,%r14 + adoxq %r15,%rax + adcxq %rax,%r15 + + + xorq %rcx,%rcx + mulxq %rbx,%rdi,%rax + adcxq %rdi,%r14 + adoxq %rax,%r15 + + mulxq %rbp,%rdi,%rax + movq %rbx,%rdx + adcxq %rdi,%r15 + adoxq %rcx,%rax + adcxq %rax,%rcx + + + mulxq %rbp,%rdi,%rbx + movq 0(%rsi),%rdx + addq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rbx + + + xorq %rbp,%rbp + adcxq %r8,%r8 + adcxq %r9,%r9 + adcxq %r10,%r10 + adcxq %r11,%r11 + adcxq %r12,%r12 + + + mulxq %rdx,%rdx,%rax + movq %rdx,0(%rdi) + movq 8(%rsi),%rdx + adoxq %rax,%r8 + movq %r8,8(%rdi) + + mulxq %rdx,%r8,%rax + movq 16(%rsi),%rdx + adoxq %r8,%r9 + adoxq %rax,%r10 + movq %r9,16(%rdi) + movq %r10,24(%rdi) + + mulxq %rdx,%r8,%r9 + movq 24(%rsi),%rdx + adoxq %r8,%r11 + adoxq %r9,%r12 + adcxq %r13,%r13 + adcxq %r14,%r14 + movq %r11,32(%rdi) + movq %r12,40(%rdi) + + mulxq %rdx,%r8,%r9 + movq 32(%rsi),%rdx + adoxq %r8,%r13 + adoxq %r9,%r14 + adcxq %r15,%r15 + adcxq %rcx,%rcx + movq %r13,48(%rdi) + movq %r14,56(%rdi) + + mulxq %rdx,%r8,%r9 + movq 40(%rsi),%rdx + adoxq %r8,%r15 + adoxq %r9,%rcx + adcxq %rbx,%rbx + adcxq %rbp,%rbp + movq %r15,64(%rdi) + movq %rcx,72(%rdi) + + mulxq %rdx,%r8,%r9 + adoxq %r8,%rbx + adoxq %r9,%rbp + + movq %rbx,80(%rdi) + movq %rbp,88(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + + + +.globl _redcx_mont_384 +.private_extern _redcx_mont_384 + +.p2align 5 +_redcx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + + + + +.globl _fromx_mont_384 +.private_extern _fromx_mont_384 + +.p2align 5 +_fromx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rdx,%rbx + call __mulx_by_1_mont_384 + + + + + movq %r14,%rax + movq %r15,%rcx + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulx_by_1_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq %rcx,%rdx + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + imulq %r8,%rdx + + + xorq %r14,%r14 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r13 + adoxq %r14,%rbp + adcxq %rbp,%r14 + imulq %r9,%rdx + + + xorq %r15,%r15 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r14 + adoxq %r15,%rbp + adcxq %rbp,%r15 + imulq %r10,%rdx + + + xorq %r8,%r8 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r10 + adoxq %rbp,%r11 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r8 + imulq %r11,%rdx + + + xorq %r9,%r9 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r11 + adoxq %rbp,%r12 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r8 + adoxq %r9,%rbp + adcxq %rbp,%r9 + imulq %r12,%rdx + + + xorq %r10,%r10 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r12 + adoxq %rbp,%r13 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r10 + imulq %r13,%rdx + + + xorq %r11,%r11 + mulxq 0(%rbx),%rax,%rbp + adcxq %rax,%r13 + adoxq %rbp,%r14 + + mulxq 8(%rbx),%rax,%rbp + adcxq %rax,%r14 + adoxq %rbp,%r15 + + mulxq 16(%rbx),%rax,%rbp + adcxq %rax,%r15 + adoxq %rbp,%r8 + + mulxq 24(%rbx),%rax,%rbp + adcxq %rax,%r8 + adoxq %rbp,%r9 + + mulxq 32(%rbx),%rax,%rbp + adcxq %rax,%r9 + adoxq %rbp,%r10 + + mulxq 40(%rbx),%rax,%rbp + movq %rcx,%rdx + adcxq %rax,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r11 + .byte 0xf3,0xc3 +.cfi_endproc + + + +.p2align 5 +__redc_tail_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + addq 48(%rsi),%r14 + movq %r14,%rax + adcq 56(%rsi),%r15 + adcq 64(%rsi),%r8 + adcq 72(%rsi),%r9 + movq %r15,%rcx + adcq 80(%rsi),%r10 + adcq 88(%rsi),%r11 + sbbq %r12,%r12 + + + + + movq %r8,%rdx + movq %r9,%rbp + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + movq %r10,%r13 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + movq %r11,%rsi + sbbq 40(%rbx),%r11 + sbbq $0,%r12 + + cmovcq %rax,%r14 + cmovcq %rcx,%r15 + cmovcq %rdx,%r8 + movq %r14,0(%rdi) + cmovcq %rbp,%r9 + movq %r15,8(%rdi) + cmovcq %r13,%r10 + movq %r8,16(%rdi) + cmovcq %rsi,%r11 + movq %r9,24(%rdi) + movq %r10,32(%rdi) + movq %r11,40(%rdi) + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sgn0x_pty_mont_384 +.private_extern _sgn0x_pty_mont_384 + +.p2align 5 +_sgn0x_pty_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 0(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + xorq %rax,%rax + movq %r14,%r13 + addq %r14,%r14 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r14 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + notq %rax + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sgn0x_pty_mont_384x +.private_extern _sgn0x_pty_mont_384x + +.p2align 5 +_sgn0x_pty_mont_384x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movq %rsi,%rbx + leaq 48(%rdi),%rsi + movq %rdx,%rcx + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + leaq 0(%rdi),%rsi + xorq %rdi,%rdi + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rdi + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rdi + + movq %r14,0(%rsp) + notq %rdi + andq $1,%r13 + andq $2,%rdi + orq %r13,%rdi + + call __mulx_by_1_mont_384 + + movq %r14,%r12 + orq %r15,%r14 + orq %r8,%r14 + orq %r9,%r14 + orq %r10,%r14 + orq %r11,%r14 + + xorq %rax,%rax + movq %r12,%r13 + addq %r12,%r12 + adcq %r15,%r15 + adcq %r8,%r8 + adcq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq $0,%rax + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r15 + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + sbbq 32(%rbx),%r10 + sbbq 40(%rbx),%r11 + sbbq $0,%rax + + movq 0(%rsp),%r12 + + notq %rax + + testq %r14,%r14 + cmovzq %rdi,%r13 + + testq %r12,%r12 + cmovnzq %rdi,%rax + + andq $1,%r13 + andq $2,%rax + orq %r13,%rax + + movq 8(%rsp),%r15 +.cfi_restore %r15 + movq 16(%rsp),%r14 +.cfi_restore %r14 + movq 24(%rsp),%r13 +.cfi_restore %r13 + movq 32(%rsp),%r12 +.cfi_restore %r12 + movq 40(%rsp),%rbx +.cfi_restore %rbx + movq 48(%rsp),%rbp +.cfi_restore %rbp + leaq 56(%rsp),%rsp +.cfi_adjust_cfa_offset -56 + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _mulx_mont_384 +.private_extern _mulx_mont_384 + +.p2align 5 +_mulx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -24(%rsp),%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + movq %r8,(%rsp) + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -8*9 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + xorq %r15,%r15 + + movq %r8,16(%rsp) + imulq 8(%rsp),%r8 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %rbp,%r15 + adoxq %rax,%r15 + adoxq %rax,%rax + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %r8,%r14 + adoxq %r8,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r9,16(%rsp) + imulq 8(%rsp),%r9 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rbp,%rax + adoxq %r8,%rax + adoxq %r8,%r8 + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r9,%r15 + adoxq %r9,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r10,16(%rsp) + imulq 8(%rsp),%r10 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %rbp,%r8 + adoxq %r9,%r8 + adoxq %r9,%r9 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r10,%rax + adoxq %r10,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r11,16(%rsp) + imulq 8(%rsp),%r11 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %rbp,%r9 + adoxq %r10,%r9 + adoxq %r10,%r10 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r11,%r8 + adoxq %r11,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + movq %r12,16(%rsp) + imulq 8(%rsp),%r12 + + + xorq %r11,%r11 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %rbp,%r10 + adoxq %r11,%r10 + adoxq %r11,%r11 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq 16(%rsp),%rdi + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r12,%r9 + adoxq %r12,%r10 + adcxq %r12,%r10 + adoxq %r12,%r11 + adcxq %r12,%r11 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + movq %r15,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + movq %rax,%rsi + + mulxq 40+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + movq %r14,%rdx + adcxq %r12,%r10 + adoxq %r12,%r11 + leaq 128(%rcx),%rcx + movq %r8,%r12 + adcq $0,%r11 + + + + + subq 0(%rcx),%r14 + sbbq 8(%rcx),%r15 + movq %r9,%rdi + sbbq 16(%rcx),%rax + sbbq 24(%rcx),%r8 + sbbq 32(%rcx),%r9 + movq %r10,%rbp + sbbq 40(%rcx),%r10 + sbbq $0,%r11 + + cmovncq %r14,%rdx + cmovcq %r13,%r15 + cmovcq %rsi,%rax + cmovncq %r8,%r12 + movq %rdx,0(%rbx) + cmovncq %r9,%rdi + movq %r15,8(%rbx) + cmovncq %r10,%rbp + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqrx_mont_384 +.private_extern _sqrx_mont_384 + +.p2align 5 +_sqrx_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -24(%rsp),%rsp +.cfi_adjust_cfa_offset 8*3 + + + movq %rcx,%r8 + leaq -128(%rdx),%rcx + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + leaq (%rsi),%rbx + movq %r8,(%rsp) + leaq -128(%rsi),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movq 24(%rsp),%r15 +.cfi_restore %r15 + movq 32(%rsp),%r14 +.cfi_restore %r14 + movq 40(%rsp),%r13 +.cfi_restore %r13 + movq 48(%rsp),%r12 +.cfi_restore %r12 + movq 56(%rsp),%rbx +.cfi_restore %rbx + movq 64(%rsp),%rbp +.cfi_restore %rbp + leaq 72(%rsp),%rsp +.cfi_adjust_cfa_offset -8*9 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqrx_n_mul_mont_384 +.private_extern _sqrx_n_mul_mont_384 + +.p2align 5 +_sqrx_n_mul_mont_384: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -40(%rsp),%rsp +.cfi_adjust_cfa_offset 8*5 + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + +L$oop_sqrx_384: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + leaq -128(%rcx),%rcx + + mulxq %rdx,%r8,%r9 + call __mulx_mont_384 + + movd %xmm1,%r10d + decl %r10d + jnz L$oop_sqrx_384 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 +.cfi_restore %r15 + movq 48(%rsp),%r14 +.cfi_restore %r14 + movq 56(%rsp),%r13 +.cfi_restore %r13 + movq 64(%rsp),%r12 +.cfi_restore %r12 + movq 72(%rsp),%rbx +.cfi_restore %rbx + movq 80(%rsp),%rbp +.cfi_restore %rbp + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -8*11 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _sqrx_n_mul_mont_383 +.private_extern _sqrx_n_mul_mont_383 + +.p2align 5 +_sqrx_n_mul_mont_383: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + leaq -40(%rsp),%rsp +.cfi_adjust_cfa_offset 8*5 + + + movq %rdx,%r10 + movq 0(%rsi),%rdx + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq %rsi,%rbx + movq 24(%rsi),%r12 + movq %rdi,16(%rsp) + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + + movq %r8,(%rsp) + movq %r9,24(%rsp) + movq 0(%r9),%xmm2 + leaq -128(%rcx),%rcx + +L$oop_sqrx_383: + movd %r10d,%xmm1 + leaq -128(%rbx),%rsi + + mulxq %rdx,%r8,%r9 + call __mulx_mont_383_nonred + + movd %xmm1,%r10d + decl %r10d + jnz L$oop_sqrx_383 + + movq %rdx,%r14 +.byte 102,72,15,126,210 + leaq -128(%rbx),%rsi + movq 24(%rsp),%rbx + + mulxq %r14,%r8,%r9 + call __mulx_mont_384 + + movq 40(%rsp),%r15 +.cfi_restore %r15 + movq 48(%rsp),%r14 +.cfi_restore %r14 + movq 56(%rsp),%r13 +.cfi_restore %r13 + movq 64(%rsp),%r12 +.cfi_restore %r12 + movq 72(%rsp),%rbx +.cfi_restore %rbx + movq 80(%rsp),%rbp +.cfi_restore %rbp + leaq 88(%rsp),%rsp +.cfi_adjust_cfa_offset -8*11 + + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 5 +__mulx_mont_383_nonred: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + mulxq %r15,%r14,%r10 + mulxq %rax,%r15,%r11 + addq %r14,%r9 + mulxq %r12,%rax,%r12 + adcq %r15,%r10 + mulxq %rdi,%rdi,%r13 + adcq %rax,%r11 + mulxq %rbp,%rbp,%r14 + movq 8(%rbx),%rdx + adcq %rdi,%r12 + adcq %rbp,%r13 + adcq $0,%r14 + movq %r8,%rax + imulq 8(%rsp),%r8 + + + xorq %r15,%r15 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r9 + adcxq %rbp,%r10 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r8,%rdx + adoxq %rdi,%r14 + adcxq %r15,%rbp + adoxq %rbp,%r15 + + + xorq %r8,%r8 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r9 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r10 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 16(%rbx),%rdx + adcxq %rdi,%r13 + adoxq %rbp,%r14 + adcxq %rax,%r14 + adoxq %rax,%r15 + adcxq %rax,%r15 + movq %r9,%r8 + imulq 8(%rsp),%r9 + + + xorq %rax,%rax + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r10 + adcxq %rbp,%r11 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r9,%rdx + adoxq %rdi,%r15 + adcxq %rax,%rbp + adoxq %rbp,%rax + + + xorq %r9,%r9 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r10 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 24(%rbx),%rdx + adcxq %rdi,%r14 + adoxq %rbp,%r15 + adcxq %r8,%r15 + adoxq %r8,%rax + adcxq %r8,%rax + movq %r10,%r9 + imulq 8(%rsp),%r10 + + + xorq %r8,%r8 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r11 + adcxq %rbp,%r12 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r10,%rdx + adoxq %rdi,%rax + adcxq %r8,%rbp + adoxq %rbp,%r8 + + + xorq %r10,%r10 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r9 + adoxq %rbp,%r11 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 40+128(%rcx),%rdi,%rbp + movq 32(%rbx),%rdx + adcxq %rdi,%r15 + adoxq %rbp,%rax + adcxq %r9,%rax + adoxq %r9,%r8 + adcxq %r9,%r8 + movq %r11,%r10 + imulq 8(%rsp),%r11 + + + xorq %r9,%r9 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r12 + adcxq %rbp,%r13 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r11,%rdx + adoxq %rdi,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r9 + + + xorq %r11,%r11 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r10 + adoxq %rbp,%r12 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 40+128(%rcx),%rdi,%rbp + movq 40(%rbx),%rdx + adcxq %rdi,%rax + adoxq %rbp,%r8 + adcxq %r10,%r8 + adoxq %r10,%r9 + adcxq %r10,%r9 + movq %r12,%r11 + imulq 8(%rsp),%r12 + + + xorq %r10,%r10 + mulxq 0+128(%rsi),%rdi,%rbp + adoxq %rdi,%r13 + adcxq %rbp,%r14 + + mulxq 8+128(%rsi),%rdi,%rbp + adoxq %rdi,%r14 + adcxq %rbp,%r15 + + mulxq 16+128(%rsi),%rdi,%rbp + adoxq %rdi,%r15 + adcxq %rbp,%rax + + mulxq 24+128(%rsi),%rdi,%rbp + adoxq %rdi,%rax + adcxq %rbp,%r8 + + mulxq 32+128(%rsi),%rdi,%rbp + adoxq %rdi,%r8 + adcxq %rbp,%r9 + + mulxq 40+128(%rsi),%rdi,%rbp + movq %r12,%rdx + adoxq %rdi,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r10 + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r11 + adoxq %rbp,%r13 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r13,%rdx + adcxq %rdi,%r8 + adoxq %rbp,%r9 + adcxq %r11,%r9 + adoxq %r11,%r10 + adcxq %r11,%r10 + imulq 8(%rsp),%rdx + movq 24(%rsp),%rbx + + + xorq %r12,%r12 + mulxq 0+128(%rcx),%rdi,%rbp + adcxq %rdi,%r13 + adoxq %rbp,%r14 + + mulxq 8+128(%rcx),%rdi,%rbp + adcxq %rdi,%r14 + adoxq %rbp,%r15 + + mulxq 16+128(%rcx),%rdi,%rbp + adcxq %rdi,%r15 + adoxq %rbp,%rax + + mulxq 24+128(%rcx),%rdi,%rbp + adcxq %rdi,%rax + adoxq %rbp,%r8 + + mulxq 32+128(%rcx),%rdi,%rbp + adcxq %rdi,%r8 + adoxq %rbp,%r9 + + mulxq 40+128(%rcx),%rdi,%rbp + movq %r14,%rdx + adcxq %rdi,%r9 + adoxq %rbp,%r10 + adcq $0,%r10 + movq %r8,%r12 + + movq %r14,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r9,%rdi + movq %r8,24(%rbx) + movq %r9,32(%rbx) + movq %r10,40(%rbx) + movq %r10,%rbp + + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _sqrx_mont_382x +.private_extern _sqrx_mont_382x + +.p2align 5 +_sqrx_mont_382x: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $136,%rsp +.cfi_adjust_cfa_offset 136 + + + movq %rcx,0(%rsp) + movq %rdx,%rcx + movq %rdi,16(%rsp) + movq %rsi,24(%rsp) + + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq 32(%rsi),%r12 + movq 40(%rsi),%r13 + + movq %r8,%r14 + addq 48(%rsi),%r8 + movq %r9,%r15 + adcq 56(%rsi),%r9 + movq %r10,%rax + adcq 64(%rsi),%r10 + movq %r11,%rdx + adcq 72(%rsi),%r11 + movq %r12,%rbx + adcq 80(%rsi),%r12 + movq %r13,%rbp + adcq 88(%rsi),%r13 + + subq 48(%rsi),%r14 + sbbq 56(%rsi),%r15 + sbbq 64(%rsi),%rax + sbbq 72(%rsi),%rdx + sbbq 80(%rsi),%rbx + sbbq 88(%rsi),%rbp + sbbq %rdi,%rdi + + movq %r8,32+0(%rsp) + movq %r9,32+8(%rsp) + movq %r10,32+16(%rsp) + movq %r11,32+24(%rsp) + movq %r12,32+32(%rsp) + movq %r13,32+40(%rsp) + + movq %r14,32+48(%rsp) + movq %r15,32+56(%rsp) + movq %rax,32+64(%rsp) + movq %rdx,32+72(%rsp) + movq %rbx,32+80(%rsp) + movq %rbp,32+88(%rsp) + movq %rdi,32+96(%rsp) + + + + leaq 48(%rsi),%rbx + + movq 48(%rsi),%rdx + movq 0(%rsi),%r14 + movq 8(%rsi),%r15 + movq 16(%rsi),%rax + movq 24(%rsi),%r12 + movq 32(%rsi),%rdi + movq 40(%rsi),%rbp + leaq -128(%rsi),%rsi + leaq -128(%rcx),%rcx + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + addq %rdx,%rdx + adcq %r15,%r15 + adcq %rax,%rax + adcq %r12,%r12 + adcq %rdi,%rdi + adcq %rbp,%rbp + + movq %rdx,48(%rbx) + movq %r15,56(%rbx) + movq %rax,64(%rbx) + movq %r12,72(%rbx) + movq %rdi,80(%rbx) + movq %rbp,88(%rbx) + + leaq 32-128(%rsp),%rsi + leaq 32+48(%rsp),%rbx + + movq 32+48(%rsp),%rdx + movq 32+0(%rsp),%r14 + movq 32+8(%rsp),%r15 + movq 32+16(%rsp),%rax + movq 32+24(%rsp),%r12 + movq 32+32(%rsp),%rdi + movq 32+40(%rsp),%rbp + + + + mulxq %r14,%r8,%r9 + call __mulx_mont_383_nonred + movq 32+96(%rsp),%r14 + leaq 128(%rcx),%rcx + movq 32+0(%rsp),%r8 + andq %r14,%r8 + movq 32+8(%rsp),%r9 + andq %r14,%r9 + movq 32+16(%rsp),%r10 + andq %r14,%r10 + movq 32+24(%rsp),%r11 + andq %r14,%r11 + movq 32+32(%rsp),%r13 + andq %r14,%r13 + andq 32+40(%rsp),%r14 + + subq %r8,%rdx + movq 0(%rcx),%r8 + sbbq %r9,%r15 + movq 8(%rcx),%r9 + sbbq %r10,%rax + movq 16(%rcx),%r10 + sbbq %r11,%r12 + movq 24(%rcx),%r11 + sbbq %r13,%rdi + movq 32(%rcx),%r13 + sbbq %r14,%rbp + sbbq %r14,%r14 + + andq %r14,%r8 + andq %r14,%r9 + andq %r14,%r10 + andq %r14,%r11 + andq %r14,%r13 + andq 40(%rcx),%r14 + + addq %r8,%rdx + adcq %r9,%r15 + adcq %r10,%rax + adcq %r11,%r12 + adcq %r13,%rdi + adcq %r14,%rbp + + movq %rdx,0(%rbx) + movq %r15,8(%rbx) + movq %rax,16(%rbx) + movq %r12,24(%rbx) + movq %rdi,32(%rbx) + movq %rbp,40(%rbx) + leaq 136(%rsp),%r8 + movq 0(%r8),%r15 +.cfi_restore %r15 + movq 8(%r8),%r14 +.cfi_restore %r14 + movq 16(%r8),%r13 +.cfi_restore %r13 + movq 24(%r8),%r12 +.cfi_restore %r12 + movq 32(%r8),%rbx +.cfi_restore %rbx + movq 40(%r8),%rbp +.cfi_restore %rbp + leaq 48(%r8),%rsp +.cfi_adjust_cfa_offset -136-8*6 + + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/sha256-armv8.S b/crypto/blst_src/build/mach-o/sha256-armv8.S new file mode 100644 index 00000000000..c928f75025f --- /dev/null +++ b/crypto/blst_src/build/mach-o/sha256-armv8.S @@ -0,0 +1,1077 @@ +// +// Copyright Supranational LLC +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// ==================================================================== +// Written by Andy Polyakov, @dot-asm, initially for the OpenSSL +// project. +// ==================================================================== +// +// sha256_block procedure for ARMv8. +// +// This module is stripped of scalar code paths, with raionale that all +// known processors are NEON-capable. +// +// See original module at CRYPTOGAMS for further details. + +.text + +.align 6 + +LK256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.long 0 //terminator + +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.align 2 +.align 2 +.globl _blst_sha256_block_armv8 + +.align 6 +_blst_sha256_block_armv8: +Lv8_entry: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ld1 {v0.4s,v1.4s},[x0] + adr x3,LK256 + +Loop_hw: + ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 + sub x2,x2,#1 + ld1 {v16.4s},[x3],#16 + rev32 v4.16b,v4.16b + rev32 v5.16b,v5.16b + rev32 v6.16b,v6.16b + rev32 v7.16b,v7.16b + orr v18.16b,v0.16b,v0.16b // offload + orr v19.16b,v1.16b,v1.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s +.long 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s +.long 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s +.long 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s +.long 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s +.long 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s +.long 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + ld1 {v17.4s},[x3] + add v16.4s,v16.4s,v6.4s + sub x3,x3,#64*4-16 // rewind + orr v2.16b,v0.16b,v0.16b +.long 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s +.long 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + add v17.4s,v17.4s,v7.4s + orr v2.16b,v0.16b,v0.16b +.long 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s +.long 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + add v0.4s,v0.4s,v18.4s + add v1.4s,v1.4s,v19.4s + + cbnz x2,Loop_hw + + st1 {v0.4s,v1.4s},[x0] + + ldr x29,[sp],#16 + ret + +.globl _blst_sha256_block_data_order + +.align 4 +_blst_sha256_block_data_order: + stp x29, x30, [sp, #-16]! + mov x29, sp + sub sp,sp,#16*4 + + adr x16,LK256 + add x2,x1,x2,lsl#6 // len to point at the end of inp + + ld1 {v0.16b},[x1], #16 + ld1 {v1.16b},[x1], #16 + ld1 {v2.16b},[x1], #16 + ld1 {v3.16b},[x1], #16 + ld1 {v4.4s},[x16], #16 + ld1 {v5.4s},[x16], #16 + ld1 {v6.4s},[x16], #16 + ld1 {v7.4s},[x16], #16 + rev32 v0.16b,v0.16b // yes, even on + rev32 v1.16b,v1.16b // big-endian + rev32 v2.16b,v2.16b + rev32 v3.16b,v3.16b + mov x17,sp + add v4.4s,v4.4s,v0.4s + add v5.4s,v5.4s,v1.4s + add v6.4s,v6.4s,v2.4s + st1 {v4.4s,v5.4s},[x17], #32 + add v7.4s,v7.4s,v3.4s + st1 {v6.4s,v7.4s},[x17] + sub x17,x17,#32 + + ldp w3,w4,[x0] + ldp w5,w6,[x0,#8] + ldp w7,w8,[x0,#16] + ldp w9,w10,[x0,#24] + ldr w12,[sp,#0] + mov w13,wzr + eor w14,w4,w5 + mov w15,wzr + b L_00_48 + +.align 4 +L_00_48: + ext v4.16b,v0.16b,v1.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v2.16b,v3.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v3.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v0.4s,v0.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v0.4s,v0.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v0.4s,v0.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v0.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v0.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v0.4s,#15 + add w8,w8,w12 + ushr v17.4s,v0.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v0.4s,#13 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v0.4s,v0.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v0.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v1.16b,v2.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v3.16b,v0.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v0.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v1.4s,v1.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v1.4s,v1.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v1.4s,v1.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v1.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v1.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v1.4s,#15 + add w4,w4,w12 + ushr v17.4s,v1.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v1.4s,#13 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v1.4s,v1.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v1.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + ext v4.16b,v2.16b,v3.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext v7.16b,v0.16b,v1.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v1.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v2.4s,v2.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v2.4s,v2.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v2.4s,v2.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v2.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v2.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v2.4s,#15 + add w8,w8,w12 + ushr v17.4s,v2.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v2.4s,#13 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v2.4s,v2.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v2.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext v4.16b,v3.16b,v0.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext v7.16b,v1.16b,v2.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v2.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v3.4s,v3.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v3.4s,v3.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v3.4s,v3.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v3.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v3.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v3.4s,#15 + add w4,w4,w12 + ushr v17.4s,v3.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v3.4s,#13 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v3.4s,v3.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v3.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[x16] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + cmp w12,#0 // check for K256 terminator + ldr w12,[sp,#0] + sub x17,x17,#64 + bne L_00_48 + + sub x16,x16,#256 // rewind x16 + cmp x1,x2 + mov x17, #64 + csel x17, x17, xzr, eq + sub x1,x1,x17 // avoid SEGV + mov x17,sp + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v0.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v0.16b,v0.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v0.4s + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v1.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v1.16b,v1.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v1.4s + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v2.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v2.16b,v2.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v2.4s + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v3.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v3.16b,v3.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v3.4s + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w3,w3,w15 // h+=Sigma0(a) from the past + ldp w11,w12,[x0,#0] + add w3,w3,w13 // h+=Maj(a,b,c) from the past + ldp w13,w14,[x0,#8] + add w3,w3,w11 // accumulate + add w4,w4,w12 + ldp w11,w12,[x0,#16] + add w5,w5,w13 + add w6,w6,w14 + ldp w13,w14,[x0,#24] + add w7,w7,w11 + add w8,w8,w12 + ldr w12,[sp,#0] + stp w3,w4,[x0,#0] + add w9,w9,w13 + mov w13,wzr + stp w5,w6,[x0,#8] + add w10,w10,w14 + stp w7,w8,[x0,#16] + eor w14,w4,w5 + stp w9,w10,[x0,#24] + mov w15,wzr + mov x17,sp + b.ne L_00_48 + + ldr x29,[x29] + add sp,sp,#16*4+16 + ret + +.globl _blst_sha256_emit +.private_extern _blst_sha256_emit + +.align 4 +_blst_sha256_emit: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] +#ifndef __AARCH64EB__ + rev x4,x4 + rev x5,x5 + rev x6,x6 + rev x7,x7 +#endif + str w4,[x0,#4] + lsr x4,x4,#32 + str w5,[x0,#12] + lsr x5,x5,#32 + str w6,[x0,#20] + lsr x6,x6,#32 + str w7,[x0,#28] + lsr x7,x7,#32 + str w4,[x0,#0] + str w5,[x0,#8] + str w6,[x0,#16] + str w7,[x0,#24] + ret + + +.globl _blst_sha256_bcopy +.private_extern _blst_sha256_bcopy + +.align 4 +_blst_sha256_bcopy: +Loop_bcopy: + ldrb w3,[x1],#1 + sub x2,x2,#1 + strb w3,[x0],#1 + cbnz x2,Loop_bcopy + ret + + +.globl _blst_sha256_hcopy +.private_extern _blst_sha256_hcopy + +.align 4 +_blst_sha256_hcopy: + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + stp x4,x5,[x0] + stp x6,x7,[x0,#16] + ret + diff --git a/crypto/blst_src/build/mach-o/sha256-portable-x86_64.s b/crypto/blst_src/build/mach-o/sha256-portable-x86_64.s new file mode 100644 index 00000000000..3f000720d00 --- /dev/null +++ b/crypto/blst_src/build/mach-o/sha256-portable-x86_64.s @@ -0,0 +1,1746 @@ +.text + +.globl _blst_sha256_block_data_order + +.p2align 4 +_blst_sha256_block_data_order: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-16 + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + shlq $4,%rdx + subq $64+24,%rsp +.cfi_adjust_cfa_offset 16*4+3*8 + leaq (%rsi,%rdx,4),%rdx + movq %rdi,64+0(%rsp) + movq %rsi,64+8(%rsp) + movq %rdx,64+16(%rsp) + + + movl 0(%rdi),%eax + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + jmp L$loop + +.p2align 4 +L$loop: + movl %ebx,%edi + leaq K256(%rip),%rbp + xorl %ecx,%edi + movl 0(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 0(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 4(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 4(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 8(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 8(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 12(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 12(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 16(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 16(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 20(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 20(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 24(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 24(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 28(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 28(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + addl %r14d,%eax + movl 32(%rsi),%r12d + movl %r8d,%r13d + movl %eax,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 32(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + addl %r14d,%r11d + movl 36(%rsi),%r12d + movl %edx,%r13d + movl %r11d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 36(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + addl %r14d,%r10d + movl 40(%rsi),%r12d + movl %ecx,%r13d + movl %r10d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 40(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + addl %r14d,%r9d + movl 44(%rsi),%r12d + movl %ebx,%r13d + movl %r9d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 44(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + addl %r14d,%r8d + movl 48(%rsi),%r12d + movl %eax,%r13d + movl %r8d,%r14d + bswapl %r12d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 48(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + addl %r14d,%edx + movl 52(%rsi),%r12d + movl %r11d,%r13d + movl %edx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 52(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + addl %r14d,%ecx + movl 56(%rsi),%r12d + movl %r10d,%r13d + movl %ecx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 56(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + addl %r14d,%ebx + movl 60(%rsi),%r12d + movl %r9d,%r13d + movl %ebx,%r14d + bswapl %r12d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 60(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + jmp L$rounds_16_xx +.p2align 4 +L$rounds_16_xx: + movl 4(%rsp),%r13d + movl 56(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 36(%rsp),%r12d + + addl 0(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,0(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 64(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 8(%rsp),%r13d + movl 60(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 40(%rsp),%r12d + + addl 4(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,4(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 68(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 12(%rsp),%r13d + movl 0(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 44(%rsp),%r12d + + addl 8(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,8(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 72(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 16(%rsp),%r13d + movl 4(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 48(%rsp),%r12d + + addl 12(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,12(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 76(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 20(%rsp),%r13d + movl 8(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 52(%rsp),%r12d + + addl 16(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,16(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 80(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 24(%rsp),%r13d + movl 12(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 56(%rsp),%r12d + + addl 20(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,20(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 84(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 28(%rsp),%r13d + movl 16(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 60(%rsp),%r12d + + addl 24(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,24(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 88(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 32(%rsp),%r13d + movl 20(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 0(%rsp),%r12d + + addl 28(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,28(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 92(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + movl 36(%rsp),%r13d + movl 24(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%eax + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 4(%rsp),%r12d + + addl 32(%rsp),%r12d + movl %r8d,%r13d + addl %r15d,%r12d + movl %eax,%r14d + rorl $14,%r13d + movl %r9d,%r15d + + xorl %r8d,%r13d + rorl $9,%r14d + xorl %r10d,%r15d + + movl %r12d,32(%rsp) + xorl %eax,%r14d + andl %r8d,%r15d + + rorl $5,%r13d + addl %r11d,%r12d + xorl %r10d,%r15d + + rorl $11,%r14d + xorl %r8d,%r13d + addl %r15d,%r12d + + movl %eax,%r15d + addl 96(%rbp),%r12d + xorl %eax,%r14d + + xorl %ebx,%r15d + rorl $6,%r13d + movl %ebx,%r11d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r11d + addl %r12d,%edx + addl %r12d,%r11d + movl 40(%rsp),%r13d + movl 28(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r11d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 8(%rsp),%r12d + + addl 36(%rsp),%r12d + movl %edx,%r13d + addl %edi,%r12d + movl %r11d,%r14d + rorl $14,%r13d + movl %r8d,%edi + + xorl %edx,%r13d + rorl $9,%r14d + xorl %r9d,%edi + + movl %r12d,36(%rsp) + xorl %r11d,%r14d + andl %edx,%edi + + rorl $5,%r13d + addl %r10d,%r12d + xorl %r9d,%edi + + rorl $11,%r14d + xorl %edx,%r13d + addl %edi,%r12d + + movl %r11d,%edi + addl 100(%rbp),%r12d + xorl %r11d,%r14d + + xorl %eax,%edi + rorl $6,%r13d + movl %eax,%r10d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r10d + addl %r12d,%ecx + addl %r12d,%r10d + movl 44(%rsp),%r13d + movl 32(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r10d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 12(%rsp),%r12d + + addl 40(%rsp),%r12d + movl %ecx,%r13d + addl %r15d,%r12d + movl %r10d,%r14d + rorl $14,%r13d + movl %edx,%r15d + + xorl %ecx,%r13d + rorl $9,%r14d + xorl %r8d,%r15d + + movl %r12d,40(%rsp) + xorl %r10d,%r14d + andl %ecx,%r15d + + rorl $5,%r13d + addl %r9d,%r12d + xorl %r8d,%r15d + + rorl $11,%r14d + xorl %ecx,%r13d + addl %r15d,%r12d + + movl %r10d,%r15d + addl 104(%rbp),%r12d + xorl %r10d,%r14d + + xorl %r11d,%r15d + rorl $6,%r13d + movl %r11d,%r9d + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%r9d + addl %r12d,%ebx + addl %r12d,%r9d + movl 48(%rsp),%r13d + movl 36(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r9d + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 16(%rsp),%r12d + + addl 44(%rsp),%r12d + movl %ebx,%r13d + addl %edi,%r12d + movl %r9d,%r14d + rorl $14,%r13d + movl %ecx,%edi + + xorl %ebx,%r13d + rorl $9,%r14d + xorl %edx,%edi + + movl %r12d,44(%rsp) + xorl %r9d,%r14d + andl %ebx,%edi + + rorl $5,%r13d + addl %r8d,%r12d + xorl %edx,%edi + + rorl $11,%r14d + xorl %ebx,%r13d + addl %edi,%r12d + + movl %r9d,%edi + addl 108(%rbp),%r12d + xorl %r9d,%r14d + + xorl %r10d,%edi + rorl $6,%r13d + movl %r10d,%r8d + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%r8d + addl %r12d,%eax + addl %r12d,%r8d + movl 52(%rsp),%r13d + movl 40(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%r8d + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 20(%rsp),%r12d + + addl 48(%rsp),%r12d + movl %eax,%r13d + addl %r15d,%r12d + movl %r8d,%r14d + rorl $14,%r13d + movl %ebx,%r15d + + xorl %eax,%r13d + rorl $9,%r14d + xorl %ecx,%r15d + + movl %r12d,48(%rsp) + xorl %r8d,%r14d + andl %eax,%r15d + + rorl $5,%r13d + addl %edx,%r12d + xorl %ecx,%r15d + + rorl $11,%r14d + xorl %eax,%r13d + addl %r15d,%r12d + + movl %r8d,%r15d + addl 112(%rbp),%r12d + xorl %r8d,%r14d + + xorl %r9d,%r15d + rorl $6,%r13d + movl %r9d,%edx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%edx + addl %r12d,%r11d + addl %r12d,%edx + movl 56(%rsp),%r13d + movl 44(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%edx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 24(%rsp),%r12d + + addl 52(%rsp),%r12d + movl %r11d,%r13d + addl %edi,%r12d + movl %edx,%r14d + rorl $14,%r13d + movl %eax,%edi + + xorl %r11d,%r13d + rorl $9,%r14d + xorl %ebx,%edi + + movl %r12d,52(%rsp) + xorl %edx,%r14d + andl %r11d,%edi + + rorl $5,%r13d + addl %ecx,%r12d + xorl %ebx,%edi + + rorl $11,%r14d + xorl %r11d,%r13d + addl %edi,%r12d + + movl %edx,%edi + addl 116(%rbp),%r12d + xorl %edx,%r14d + + xorl %r8d,%edi + rorl $6,%r13d + movl %r8d,%ecx + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%ecx + addl %r12d,%r10d + addl %r12d,%ecx + movl 60(%rsp),%r13d + movl 48(%rsp),%r15d + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ecx + movl %r15d,%r14d + rorl $2,%r15d + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%r15d + shrl $10,%r14d + + rorl $17,%r15d + xorl %r13d,%r12d + xorl %r14d,%r15d + addl 28(%rsp),%r12d + + addl 56(%rsp),%r12d + movl %r10d,%r13d + addl %r15d,%r12d + movl %ecx,%r14d + rorl $14,%r13d + movl %r11d,%r15d + + xorl %r10d,%r13d + rorl $9,%r14d + xorl %eax,%r15d + + movl %r12d,56(%rsp) + xorl %ecx,%r14d + andl %r10d,%r15d + + rorl $5,%r13d + addl %ebx,%r12d + xorl %eax,%r15d + + rorl $11,%r14d + xorl %r10d,%r13d + addl %r15d,%r12d + + movl %ecx,%r15d + addl 120(%rbp),%r12d + xorl %ecx,%r14d + + xorl %edx,%r15d + rorl $6,%r13d + movl %edx,%ebx + + andl %r15d,%edi + rorl $2,%r14d + addl %r13d,%r12d + + xorl %edi,%ebx + addl %r12d,%r9d + addl %r12d,%ebx + movl 0(%rsp),%r13d + movl 52(%rsp),%edi + + movl %r13d,%r12d + rorl $11,%r13d + addl %r14d,%ebx + movl %edi,%r14d + rorl $2,%edi + + xorl %r12d,%r13d + shrl $3,%r12d + rorl $7,%r13d + xorl %r14d,%edi + shrl $10,%r14d + + rorl $17,%edi + xorl %r13d,%r12d + xorl %r14d,%edi + addl 32(%rsp),%r12d + + addl 60(%rsp),%r12d + movl %r9d,%r13d + addl %edi,%r12d + movl %ebx,%r14d + rorl $14,%r13d + movl %r10d,%edi + + xorl %r9d,%r13d + rorl $9,%r14d + xorl %r11d,%edi + + movl %r12d,60(%rsp) + xorl %ebx,%r14d + andl %r9d,%edi + + rorl $5,%r13d + addl %eax,%r12d + xorl %r11d,%edi + + rorl $11,%r14d + xorl %r9d,%r13d + addl %edi,%r12d + + movl %ebx,%edi + addl 124(%rbp),%r12d + xorl %ebx,%r14d + + xorl %ecx,%edi + rorl $6,%r13d + movl %ecx,%eax + + andl %edi,%r15d + rorl $2,%r14d + addl %r13d,%r12d + + xorl %r15d,%eax + addl %r12d,%r8d + addl %r12d,%eax + leaq 64(%rbp),%rbp + cmpb $0x19,3(%rbp) + jnz L$rounds_16_xx + + movq 64+0(%rsp),%rdi + addl %r14d,%eax + leaq 64(%rsi),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + cmpq 64+16(%rsp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb L$loop + + leaq 64+24+48(%rsp),%r11 +.cfi_def_cfa %r11,8 + movq 64+24(%rsp),%r15 +.cfi_restore %r15 + movq -40(%r11),%r14 +.cfi_restore %r14 + movq -32(%r11),%r13 +.cfi_restore %r13 + movq -24(%r11),%r12 +.cfi_restore %r12 + movq -16(%r11),%rbp +.cfi_restore %rbp + movq -8(%r11),%rbx +.cfi_restore %rbx + + leaq (%r11),%rsp + .byte 0xf3,0xc3 +.cfi_endproc + + +.p2align 6 + +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl _blst_sha256_emit +.private_extern _blst_sha256_emit + +.p2align 4 +_blst_sha256_emit: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + bswapq %r8 + movq 24(%rsi),%r11 + bswapq %r9 + movl %r8d,4(%rdi) + bswapq %r10 + movl %r9d,12(%rdi) + bswapq %r11 + movl %r10d,20(%rdi) + shrq $32,%r8 + movl %r11d,28(%rdi) + shrq $32,%r9 + movl %r8d,0(%rdi) + shrq $32,%r10 + movl %r9d,8(%rdi) + shrq $32,%r11 + movl %r10d,16(%rdi) + movl %r11d,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _blst_sha256_bcopy +.private_extern _blst_sha256_bcopy + +.p2align 4 +_blst_sha256_bcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rsi,%rdi +L$oop_bcopy: + movzbl (%rsi),%eax + leaq 1(%rsi),%rsi + movb %al,-1(%rdi,%rsi,1) + decq %rdx + jnz L$oop_bcopy + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _blst_sha256_hcopy +.private_extern _blst_sha256_hcopy + +.p2align 4 +_blst_sha256_hcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/mach-o/sha256-x86_64.s b/crypto/blst_src/build/mach-o/sha256-x86_64.s new file mode 100644 index 00000000000..dee75e35362 --- /dev/null +++ b/crypto/blst_src/build/mach-o/sha256-x86_64.s @@ -0,0 +1,1438 @@ +.text + +.p2align 6 + +K256: +.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f +.long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff +.long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908 +.byte 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,64,100,111,116,45,97,115,109,0 +.globl _blst_sha256_block_data_order_shaext +.private_extern _blst_sha256_block_data_order_shaext + +.p2align 6 +_blst_sha256_block_data_order_shaext: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + leaq K256+128(%rip),%rcx + movdqu (%rdi),%xmm1 + movdqu 16(%rdi),%xmm2 + movdqa 256-128(%rcx),%xmm7 + + pshufd $0x1b,%xmm1,%xmm0 + pshufd $0xb1,%xmm1,%xmm1 + pshufd $0x1b,%xmm2,%xmm2 + movdqa %xmm7,%xmm8 +.byte 102,15,58,15,202,8 + punpcklqdq %xmm0,%xmm2 + jmp L$oop_shaext + +.p2align 4 +L$oop_shaext: + movdqu (%rsi),%xmm3 + movdqu 16(%rsi),%xmm4 + movdqu 32(%rsi),%xmm5 +.byte 102,15,56,0,223 + movdqu 48(%rsi),%xmm6 + + movdqa 0-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 102,15,56,0,231 + movdqa %xmm2,%xmm10 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + nop + movdqa %xmm1,%xmm9 +.byte 15,56,203,202 + + movdqa 16-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 102,15,56,0,239 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + leaq 64(%rsi),%rsi +.byte 15,56,204,220 +.byte 15,56,203,202 + + movdqa 32-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 102,15,56,0,247 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + + movdqa 48-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 64-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 80-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 96-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 112-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 128-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 144-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 + nop + paddd %xmm7,%xmm6 +.byte 15,56,204,220 +.byte 15,56,203,202 + movdqa 160-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,205,245 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm6,%xmm7 +.byte 102,15,58,15,253,4 + nop + paddd %xmm7,%xmm3 +.byte 15,56,204,229 +.byte 15,56,203,202 + movdqa 176-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 +.byte 15,56,205,222 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm3,%xmm7 +.byte 102,15,58,15,254,4 + nop + paddd %xmm7,%xmm4 +.byte 15,56,204,238 +.byte 15,56,203,202 + movdqa 192-128(%rcx),%xmm0 + paddd %xmm3,%xmm0 +.byte 15,56,205,227 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm4,%xmm7 +.byte 102,15,58,15,251,4 + nop + paddd %xmm7,%xmm5 +.byte 15,56,204,243 +.byte 15,56,203,202 + movdqa 208-128(%rcx),%xmm0 + paddd %xmm4,%xmm0 +.byte 15,56,205,236 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + movdqa %xmm5,%xmm7 +.byte 102,15,58,15,252,4 +.byte 15,56,203,202 + paddd %xmm7,%xmm6 + + movdqa 224-128(%rcx),%xmm0 + paddd %xmm5,%xmm0 +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 +.byte 15,56,205,245 + movdqa %xmm8,%xmm7 +.byte 15,56,203,202 + + movdqa 240-128(%rcx),%xmm0 + paddd %xmm6,%xmm0 + nop +.byte 15,56,203,209 + pshufd $0x0e,%xmm0,%xmm0 + decq %rdx + nop +.byte 15,56,203,202 + + paddd %xmm10,%xmm2 + paddd %xmm9,%xmm1 + jnz L$oop_shaext + + pshufd $0xb1,%xmm2,%xmm2 + pshufd $0x1b,%xmm1,%xmm7 + pshufd $0xb1,%xmm1,%xmm1 + punpckhqdq %xmm2,%xmm1 +.byte 102,15,58,15,215,8 + + movdqu %xmm1,(%rdi) + movdqu %xmm2,16(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _blst_sha256_block_data_order +.private_extern _blst_sha256_block_data_order + +.p2align 6 +_blst_sha256_block_data_order: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + shlq $4,%rdx + subq $40,%rsp +.cfi_adjust_cfa_offset 40 + leaq (%rsi,%rdx,4),%rdx + movq %rdi,0(%rsp) + + movq %rdx,16(%rsp) + movq %rsp,%rbp +.cfi_def_cfa_register %rbp + + + leaq -64(%rsp),%rsp + movl 0(%rdi),%eax + andq $-64,%rsp + movl 4(%rdi),%ebx + movl 8(%rdi),%ecx + movl 12(%rdi),%edx + movl 16(%rdi),%r8d + movl 20(%rdi),%r9d + movl 24(%rdi),%r10d + movl 28(%rdi),%r11d + + + jmp L$loop_ssse3 +.p2align 4 +L$loop_ssse3: + movdqa K256+256(%rip),%xmm7 + movq %rsi,8(%rbp) + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 +.byte 102,15,56,0,199 + movdqu 48(%rsi),%xmm3 + leaq K256(%rip),%rsi +.byte 102,15,56,0,207 + movdqa 0(%rsi),%xmm4 + movdqa 16(%rsi),%xmm5 +.byte 102,15,56,0,215 + paddd %xmm0,%xmm4 + movdqa 32(%rsi),%xmm6 +.byte 102,15,56,0,223 + movdqa 48(%rsi),%xmm7 + paddd %xmm1,%xmm5 + paddd %xmm2,%xmm6 + paddd %xmm3,%xmm7 + movdqa %xmm4,0(%rsp) + movl %eax,%r14d + movdqa %xmm5,16(%rsp) + movl %ebx,%edi + movdqa %xmm6,32(%rsp) + xorl %ecx,%edi + movdqa %xmm7,48(%rsp) + movl %r8d,%r13d + jmp L$ssse3_00_47 + +.p2align 4 +L$ssse3_00_47: + subq $-64,%rsi + rorl $14,%r13d + movdqa %xmm1,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm3,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,224,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,250,4 + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm3,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 4(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm0 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm0 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm0,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 0(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm0 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm0,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,0(%rsp) + rorl $14,%r13d + movdqa %xmm2,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm0,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,225,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,251,4 + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm0,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 20(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm1 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm1 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm1,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 16(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm1 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm1,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,16(%rsp) + rorl $14,%r13d + movdqa %xmm3,%xmm4 + movl %r14d,%eax + movl %r9d,%r12d + movdqa %xmm1,%xmm7 + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d +.byte 102,15,58,15,226,4 + andl %r8d,%r12d + xorl %r8d,%r13d +.byte 102,15,58,15,248,4 + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %ebx,%r15d + addl %r12d,%r11d + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r11d,%edx + psrld $7,%xmm6 + addl %edi,%r11d + movl %edx,%r13d + pshufd $250,%xmm1,%xmm7 + addl %r11d,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%r11d + movl %r8d,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %r11d,%r14d + pxor %xmm5,%xmm4 + andl %edx,%r12d + xorl %edx,%r13d + pslld $11,%xmm5 + addl 36(%rsp),%r10d + movl %r11d,%edi + pxor %xmm6,%xmm4 + xorl %r9d,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %eax,%edi + addl %r12d,%r10d + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + psrld $10,%xmm7 + addl %r13d,%r10d + xorl %eax,%r15d + paddd %xmm4,%xmm2 + rorl $2,%r14d + addl %r10d,%ecx + psrlq $17,%xmm6 + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %ecx,%r13d + xorl %r8d,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + psrldq $8,%xmm7 + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + paddd %xmm7,%xmm2 + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + pshufd $80,%xmm2,%xmm7 + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + movdqa %xmm7,%xmm6 + addl %edi,%r9d + movl %ebx,%r13d + psrld $10,%xmm7 + addl %r9d,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%r9d + movl %ecx,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + psrlq $2,%xmm6 + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + pxor %xmm6,%xmm7 + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %r10d,%edi + addl %r12d,%r8d + movdqa 32(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + paddd %xmm7,%xmm2 + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + paddd %xmm2,%xmm6 + movl %eax,%r13d + addl %r8d,%r14d + movdqa %xmm6,32(%rsp) + rorl $14,%r13d + movdqa %xmm0,%xmm4 + movl %r14d,%r8d + movl %ebx,%r12d + movdqa %xmm2,%xmm7 + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d +.byte 102,15,58,15,227,4 + andl %eax,%r12d + xorl %eax,%r13d +.byte 102,15,58,15,249,4 + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + movdqa %xmm4,%xmm5 + xorl %r9d,%r15d + addl %r12d,%edx + movdqa %xmm4,%xmm6 + rorl $6,%r13d + andl %r15d,%edi + psrld $3,%xmm4 + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %edx,%r11d + psrld $7,%xmm6 + addl %edi,%edx + movl %r11d,%r13d + pshufd $250,%xmm2,%xmm7 + addl %edx,%r14d + rorl $14,%r13d + pslld $14,%xmm5 + movl %r14d,%edx + movl %eax,%r12d + pxor %xmm6,%xmm4 + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + psrld $11,%xmm6 + xorl %edx,%r14d + pxor %xmm5,%xmm4 + andl %r11d,%r12d + xorl %r11d,%r13d + pslld $11,%xmm5 + addl 52(%rsp),%ecx + movl %edx,%edi + pxor %xmm6,%xmm4 + xorl %ebx,%r12d + rorl $11,%r14d + movdqa %xmm7,%xmm6 + xorl %r8d,%edi + addl %r12d,%ecx + pxor %xmm5,%xmm4 + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + psrld $10,%xmm7 + addl %r13d,%ecx + xorl %r8d,%r15d + paddd %xmm4,%xmm3 + rorl $2,%r14d + addl %ecx,%r10d + psrlq $17,%xmm6 + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + pxor %xmm6,%xmm7 + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + psrlq $2,%xmm6 + xorl %r10d,%r13d + xorl %eax,%r12d + pxor %xmm6,%xmm7 + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + pshufd $128,%xmm7,%xmm7 + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + psrldq $8,%xmm7 + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + paddd %xmm7,%xmm3 + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + pshufd $80,%xmm3,%xmm7 + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + movdqa %xmm7,%xmm6 + addl %edi,%ebx + movl %r9d,%r13d + psrld $10,%xmm7 + addl %ebx,%r14d + rorl $14,%r13d + psrlq $17,%xmm6 + movl %r14d,%ebx + movl %r10d,%r12d + pxor %xmm6,%xmm7 + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + psrlq $2,%xmm6 + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + pxor %xmm6,%xmm7 + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + pshufd $8,%xmm7,%xmm7 + xorl %ecx,%edi + addl %r12d,%eax + movdqa 48(%rsi),%xmm6 + rorl $6,%r13d + andl %edi,%r15d + pslldq $8,%xmm7 + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + paddd %xmm7,%xmm3 + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + paddd %xmm3,%xmm6 + movl %r8d,%r13d + addl %eax,%r14d + movdqa %xmm6,48(%rsp) + cmpb $0,67(%rsi) + jne L$ssse3_00_47 + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 0(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 4(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 8(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 12(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 16(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 20(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 24(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 28(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + rorl $14,%r13d + movl %r14d,%eax + movl %r9d,%r12d + rorl $9,%r14d + xorl %r8d,%r13d + xorl %r10d,%r12d + rorl $5,%r13d + xorl %eax,%r14d + andl %r8d,%r12d + xorl %r8d,%r13d + addl 32(%rsp),%r11d + movl %eax,%r15d + xorl %r10d,%r12d + rorl $11,%r14d + xorl %ebx,%r15d + addl %r12d,%r11d + rorl $6,%r13d + andl %r15d,%edi + xorl %eax,%r14d + addl %r13d,%r11d + xorl %ebx,%edi + rorl $2,%r14d + addl %r11d,%edx + addl %edi,%r11d + movl %edx,%r13d + addl %r11d,%r14d + rorl $14,%r13d + movl %r14d,%r11d + movl %r8d,%r12d + rorl $9,%r14d + xorl %edx,%r13d + xorl %r9d,%r12d + rorl $5,%r13d + xorl %r11d,%r14d + andl %edx,%r12d + xorl %edx,%r13d + addl 36(%rsp),%r10d + movl %r11d,%edi + xorl %r9d,%r12d + rorl $11,%r14d + xorl %eax,%edi + addl %r12d,%r10d + rorl $6,%r13d + andl %edi,%r15d + xorl %r11d,%r14d + addl %r13d,%r10d + xorl %eax,%r15d + rorl $2,%r14d + addl %r10d,%ecx + addl %r15d,%r10d + movl %ecx,%r13d + addl %r10d,%r14d + rorl $14,%r13d + movl %r14d,%r10d + movl %edx,%r12d + rorl $9,%r14d + xorl %ecx,%r13d + xorl %r8d,%r12d + rorl $5,%r13d + xorl %r10d,%r14d + andl %ecx,%r12d + xorl %ecx,%r13d + addl 40(%rsp),%r9d + movl %r10d,%r15d + xorl %r8d,%r12d + rorl $11,%r14d + xorl %r11d,%r15d + addl %r12d,%r9d + rorl $6,%r13d + andl %r15d,%edi + xorl %r10d,%r14d + addl %r13d,%r9d + xorl %r11d,%edi + rorl $2,%r14d + addl %r9d,%ebx + addl %edi,%r9d + movl %ebx,%r13d + addl %r9d,%r14d + rorl $14,%r13d + movl %r14d,%r9d + movl %ecx,%r12d + rorl $9,%r14d + xorl %ebx,%r13d + xorl %edx,%r12d + rorl $5,%r13d + xorl %r9d,%r14d + andl %ebx,%r12d + xorl %ebx,%r13d + addl 44(%rsp),%r8d + movl %r9d,%edi + xorl %edx,%r12d + rorl $11,%r14d + xorl %r10d,%edi + addl %r12d,%r8d + rorl $6,%r13d + andl %edi,%r15d + xorl %r9d,%r14d + addl %r13d,%r8d + xorl %r10d,%r15d + rorl $2,%r14d + addl %r8d,%eax + addl %r15d,%r8d + movl %eax,%r13d + addl %r8d,%r14d + rorl $14,%r13d + movl %r14d,%r8d + movl %ebx,%r12d + rorl $9,%r14d + xorl %eax,%r13d + xorl %ecx,%r12d + rorl $5,%r13d + xorl %r8d,%r14d + andl %eax,%r12d + xorl %eax,%r13d + addl 48(%rsp),%edx + movl %r8d,%r15d + xorl %ecx,%r12d + rorl $11,%r14d + xorl %r9d,%r15d + addl %r12d,%edx + rorl $6,%r13d + andl %r15d,%edi + xorl %r8d,%r14d + addl %r13d,%edx + xorl %r9d,%edi + rorl $2,%r14d + addl %edx,%r11d + addl %edi,%edx + movl %r11d,%r13d + addl %edx,%r14d + rorl $14,%r13d + movl %r14d,%edx + movl %eax,%r12d + rorl $9,%r14d + xorl %r11d,%r13d + xorl %ebx,%r12d + rorl $5,%r13d + xorl %edx,%r14d + andl %r11d,%r12d + xorl %r11d,%r13d + addl 52(%rsp),%ecx + movl %edx,%edi + xorl %ebx,%r12d + rorl $11,%r14d + xorl %r8d,%edi + addl %r12d,%ecx + rorl $6,%r13d + andl %edi,%r15d + xorl %edx,%r14d + addl %r13d,%ecx + xorl %r8d,%r15d + rorl $2,%r14d + addl %ecx,%r10d + addl %r15d,%ecx + movl %r10d,%r13d + addl %ecx,%r14d + rorl $14,%r13d + movl %r14d,%ecx + movl %r11d,%r12d + rorl $9,%r14d + xorl %r10d,%r13d + xorl %eax,%r12d + rorl $5,%r13d + xorl %ecx,%r14d + andl %r10d,%r12d + xorl %r10d,%r13d + addl 56(%rsp),%ebx + movl %ecx,%r15d + xorl %eax,%r12d + rorl $11,%r14d + xorl %edx,%r15d + addl %r12d,%ebx + rorl $6,%r13d + andl %r15d,%edi + xorl %ecx,%r14d + addl %r13d,%ebx + xorl %edx,%edi + rorl $2,%r14d + addl %ebx,%r9d + addl %edi,%ebx + movl %r9d,%r13d + addl %ebx,%r14d + rorl $14,%r13d + movl %r14d,%ebx + movl %r10d,%r12d + rorl $9,%r14d + xorl %r9d,%r13d + xorl %r11d,%r12d + rorl $5,%r13d + xorl %ebx,%r14d + andl %r9d,%r12d + xorl %r9d,%r13d + addl 60(%rsp),%eax + movl %ebx,%edi + xorl %r11d,%r12d + rorl $11,%r14d + xorl %ecx,%edi + addl %r12d,%eax + rorl $6,%r13d + andl %edi,%r15d + xorl %ebx,%r14d + addl %r13d,%eax + xorl %ecx,%r15d + rorl $2,%r14d + addl %eax,%r8d + addl %r15d,%eax + movl %r8d,%r13d + addl %eax,%r14d + movq 0(%rbp),%rdi + movl %r14d,%eax + movq 8(%rbp),%rsi + + addl 0(%rdi),%eax + addl 4(%rdi),%ebx + addl 8(%rdi),%ecx + addl 12(%rdi),%edx + addl 16(%rdi),%r8d + addl 20(%rdi),%r9d + addl 24(%rdi),%r10d + addl 28(%rdi),%r11d + + leaq 64(%rsi),%rsi + cmpq 16(%rbp),%rsi + + movl %eax,0(%rdi) + movl %ebx,4(%rdi) + movl %ecx,8(%rdi) + movl %edx,12(%rdi) + movl %r8d,16(%rdi) + movl %r9d,20(%rdi) + movl %r10d,24(%rdi) + movl %r11d,28(%rdi) + jb L$loop_ssse3 + + xorps %xmm0,%xmm0 + leaq 40+48(%rbp),%r11 +.cfi_def_cfa %r11,8 + movaps %xmm0,0(%rsp) + movaps %xmm0,16(%rsp) + movaps %xmm0,32(%rsp) + movaps %xmm0,48(%rsp) + movq 40(%rbp),%r15 +.cfi_restore %r15 + movq -40(%r11),%r14 +.cfi_restore %r14 + movq -32(%r11),%r13 +.cfi_restore %r13 + movq -24(%r11),%r12 +.cfi_restore %r12 + movq -16(%r11),%rbx +.cfi_restore %rbx + movq -8(%r11),%rbp +.cfi_restore %rbp + + leaq (%r11),%rsp + .byte 0xf3,0xc3 +.cfi_endproc + +.globl _blst_sha256_emit +.private_extern _blst_sha256_emit + +.p2align 4 +_blst_sha256_emit: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + bswapq %r8 + movq 24(%rsi),%r11 + bswapq %r9 + movl %r8d,4(%rdi) + bswapq %r10 + movl %r9d,12(%rdi) + bswapq %r11 + movl %r10d,20(%rdi) + shrq $32,%r8 + movl %r11d,28(%rdi) + shrq $32,%r9 + movl %r8d,0(%rdi) + shrq $32,%r10 + movl %r9d,8(%rdi) + shrq $32,%r11 + movl %r10d,16(%rdi) + movl %r11d,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _blst_sha256_bcopy +.private_extern _blst_sha256_bcopy + +.p2align 4 +_blst_sha256_bcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + subq %rsi,%rdi +L$oop_bcopy: + movzbl (%rsi),%eax + leaq 1(%rsi),%rsi + movb %al,-1(%rdi,%rsi,1) + decq %rdx + jnz L$oop_bcopy + .byte 0xf3,0xc3 +.cfi_endproc + + +.globl _blst_sha256_hcopy +.private_extern _blst_sha256_hcopy + +.p2align 4 +_blst_sha256_hcopy: +.cfi_startproc + .byte 0xf3,0x0f,0x1e,0xfa + + movq 0(%rsi),%r8 + movq 8(%rsi),%r9 + movq 16(%rsi),%r10 + movq 24(%rsi),%r11 + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + .byte 0xf3,0xc3 +.cfi_endproc + diff --git a/crypto/blst_src/build/win64/add_mod_256-armv8.asm b/crypto/blst_src/build/win64/add_mod_256-armv8.asm new file mode 100644 index 00000000000..8d6975185a6 --- /dev/null +++ b/crypto/blst_src/build/win64/add_mod_256-armv8.asm @@ -0,0 +1,380 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + + EXPORT |add_mod_256|[FUNC] + ALIGN 32 +|add_mod_256| PROC + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + adds x8,x8,x12 + ldp x14,x15,[x2,#16] + adcs x9,x9,x13 + ldp x4,x5,[x3] + adcs x10,x10,x14 + ldp x6,x7,[x3,#16] + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csello x8,x8,x16 + csello x9,x9,x17 + csello x10,x10,x1 + stp x8,x9,[x0] + csello x11,x11,x2 + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |mul_by_3_mod_256|[FUNC] + ALIGN 32 +|mul_by_3_mod_256| PROC + ldp x12,x13,[x1] + ldp x14,x15,[x1,#16] + + adds x8,x12,x12 + ldp x4,x5,[x2] + adcs x9,x13,x13 + ldp x6,x7,[x2,#16] + adcs x10,x14,x14 + adcs x11,x15,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csello x8,x8,x16 + csello x9,x9,x17 + csello x10,x10,x1 + csello x11,x11,x2 + + adds x8,x8,x12 + adcs x9,x9,x13 + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csello x8,x8,x16 + csello x9,x9,x17 + csello x10,x10,x1 + stp x8,x9,[x0] + csello x11,x11,x2 + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |lshift_mod_256|[FUNC] + ALIGN 32 +|lshift_mod_256| PROC + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +|$Loop_lshift_mod_256| + adds x8,x8,x8 + sub x2,x2,#1 + adcs x9,x9,x9 + adcs x10,x10,x10 + adcs x11,x11,x11 + adc x3,xzr,xzr + + subs x12,x8,x4 + sbcs x13,x9,x5 + sbcs x14,x10,x6 + sbcs x15,x11,x7 + sbcs xzr,x3,xzr + + csello x8,x8,x12 + csello x9,x9,x13 + csello x10,x10,x14 + csello x11,x11,x15 + + cbnz x2,|$Loop_lshift_mod_256| + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |rshift_mod_256|[FUNC] + ALIGN 32 +|rshift_mod_256| PROC + ldp x8,x9,[x1] + ldp x10,x11,[x1,#16] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + +|$Loop_rshift| + adds x12,x8,x4 + sub x2,x2,#1 + adcs x13,x9,x5 + adcs x14,x10,x6 + adcs x15,x11,x7 + adc x3,xzr,xzr + tst x8,#1 + + cselne x12,x12,x8 + cselne x13,x13,x9 + cselne x14,x14,x10 + cselne x15,x15,x11 + cselne x3,x3,xzr + + extr x8,x13,x12,#1 + extr x9,x14,x13,#1 + extr x10,x15,x14,#1 + extr x11,x3,x15,#1 + + cbnz x2,|$Loop_rshift| + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |cneg_mod_256|[FUNC] + ALIGN 32 +|cneg_mod_256| PROC + ldp x8,x9,[x1] + ldp x4,x5,[x3] + + ldp x10,x11,[x1,#16] + subs x12,x4,x8 + ldp x6,x7,[x3,#16] + orr x4,x8,x9 + sbcs x13,x5,x9 + orr x5,x10,x11 + sbcs x14,x6,x10 + orr x3,x4,x5 + sbc x15,x7,x11 + + cmp x3,#0 + csetmne x3 + ands x2,x2,x3 + + cseleq x8,x8,x12 + cseleq x9,x9,x13 + cseleq x10,x10,x14 + stp x8,x9,[x0] + cseleq x11,x11,x15 + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |sub_mod_256|[FUNC] + ALIGN 32 +|sub_mod_256| PROC + ldp x8,x9,[x1] + ldp x12,x13,[x2] + + ldp x10,x11,[x1,#16] + subs x8,x8,x12 + ldp x14,x15,[x2,#16] + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + stp x8,x9,[x0] + adc x11,x11,x7 + stp x10,x11,[x0,#16] + + ret + ENDP + + + + EXPORT |check_mod_256|[FUNC] + ALIGN 32 +|check_mod_256| PROC + ldp x8,x9,[x0] + ldp x10,x11,[x0,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + subs xzr,x8,x4 + sbcs xzr,x9,x5 + orr x8,x8,x9 + sbcs xzr,x10,x6 + orr x8,x8,x10 + sbcs xzr,x11,x7 + orr x8,x8,x11 + sbc x1,xzr,xzr + + cmp x8,#0 + mov x0,#1 + cselne x0,x0,xzr + and x0,x0,x1 + + ret + ENDP + + + + EXPORT |add_n_check_mod_256|[FUNC] + ALIGN 32 +|add_n_check_mod_256| PROC + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + adds x8,x8,x12 + ldp x4,x5,[x3] + adcs x9,x9,x13 + ldp x6,x7,[x3,#16] + adcs x10,x10,x14 + adcs x11,x11,x15 + adc x3,xzr,xzr + + subs x16,x8,x4 + sbcs x17,x9,x5 + sbcs x1,x10,x6 + sbcs x2,x11,x7 + sbcs xzr,x3,xzr + + csello x8,x8,x16 + csello x9,x9,x17 + csello x10,x10,x1 + csello x11,x11,x2 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + cselne x0,x17,xzr + + ret + ENDP + + + + EXPORT |sub_n_check_mod_256|[FUNC] + ALIGN 32 +|sub_n_check_mod_256| PROC + ldp x8,x9,[x1] + ldp x12,x13,[x2] + ldp x10,x11,[x1,#16] + ldp x14,x15,[x2,#16] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 + rev x10,x10 + rev x14,x14 + rev x11,x11 + rev x15,x15 +#endif + + subs x8,x8,x12 + sbcs x9,x9,x13 + ldp x4,x5,[x3] + sbcs x10,x10,x14 + ldp x6,x7,[x3,#16] + sbcs x11,x11,x15 + sbc x3,xzr,xzr + + and x4,x4,x3 + and x5,x5,x3 + adds x8,x8,x4 + and x6,x6,x3 + adcs x9,x9,x5 + and x7,x7,x3 + adcs x10,x10,x6 + adc x11,x11,x7 + + orr x16, x8, x9 + orr x17, x10, x11 + orr x16, x16, x17 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x9,x9 + rev x10,x10 + rev x11,x11 +#endif + + stp x8,x9,[x0] + stp x10,x11,[x0,#16] + + mov x17, #1 + cmp x16, #0 + cselne x0,x17,xzr + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/add_mod_256-x86_64.asm b/crypto/blst_src/build/win64/add_mod_256-x86_64.asm new file mode 100644 index 00000000000..09a5c17975d --- /dev/null +++ b/crypto/blst_src/build/win64/add_mod_256-x86_64.asm @@ -0,0 +1,934 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC add_mod_256 + + +ALIGN 32 +add_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_add_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_add_mod_256:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + +$L$oaded_a_add_mod_256:: + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + mov rax,r8 + adc r10,QWORD PTR[16+rdx] + mov rsi,r9 + adc r11,QWORD PTR[24+rdx] + sbb rdx,rdx + + mov rbx,r10 + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rcx] + mov rbp,r11 + sbb r11,QWORD PTR[24+rcx] + sbb rdx,0 + + cmovc r8,rax + cmovc r9,rsi + mov QWORD PTR[rdi],r8 + cmovc r10,rbx + mov QWORD PTR[8+rdi],r9 + cmovc r11,rbp + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_add_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_add_mod_256:: +add_mod_256 ENDP + + +PUBLIC mul_by_3_mod_256 + + +ALIGN 32 +mul_by_3_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_3_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + +$L$SEH_body_mul_by_3_mod_256:: + + + mov rcx,rdx + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov rdx,rsi + mov r11,QWORD PTR[24+rsi] + + call __lshift_mod_256 + mov r12,QWORD PTR[rsp] + + jmp $L$oaded_a_add_mod_256 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_mul_by_3_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_3_mod_256:: +mul_by_3_mod_256 ENDP + + +ALIGN 32 +__lshift_mod_256 PROC PRIVATE + DB 243,15,30,250 + add r8,r8 + adc r9,r9 + mov rax,r8 + adc r10,r10 + mov rsi,r9 + adc r11,r11 + sbb r12,r12 + + mov rbx,r10 + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rcx] + mov rbp,r11 + sbb r11,QWORD PTR[24+rcx] + sbb r12,0 + + cmovc r8,rax + cmovc r9,rsi + cmovc r10,rbx + cmovc r11,rbp + + DB 0F3h,0C3h ;repret +__lshift_mod_256 ENDP + + +PUBLIC lshift_mod_256 + + +ALIGN 32 +lshift_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_lshift_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + +$L$SEH_body_lshift_mod_256:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + +$L$oop_lshift_mod_256:: + call __lshift_mod_256 + dec edx + jnz $L$oop_lshift_mod_256 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + mov r12,QWORD PTR[rsp] + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_lshift_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_lshift_mod_256:: +lshift_mod_256 ENDP + + +PUBLIC rshift_mod_256 + + +ALIGN 32 +rshift_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_rshift_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_rshift_mod_256:: + + + mov rbp,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + +$L$oop_rshift_mod_256:: + mov r8,rbp + and rbp,1 + mov rax,QWORD PTR[rcx] + neg rbp + mov rsi,QWORD PTR[8+rcx] + mov rbx,QWORD PTR[16+rcx] + + and rax,rbp + and rsi,rbp + and rbx,rbp + and rbp,QWORD PTR[24+rcx] + + add r8,rax + adc r9,rsi + adc r10,rbx + adc r11,rbp + sbb rax,rax + + shr r8,1 + mov rbp,r9 + shr r9,1 + mov rbx,r10 + shr r10,1 + mov rsi,r11 + shr r11,1 + + shl rbp,63 + shl rbx,63 + or rbp,r8 + shl rsi,63 + or r9,rbx + shl rax,63 + or r10,rsi + or r11,rax + + dec edx + jnz $L$oop_rshift_mod_256 + + mov QWORD PTR[rdi],rbp + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_rshift_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_rshift_mod_256:: +rshift_mod_256 ENDP + + +PUBLIC cneg_mod_256 + + +ALIGN 32 +cneg_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_cneg_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + +$L$SEH_body_cneg_mod_256:: + + + mov r12,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r8,r12 + mov r11,QWORD PTR[24+rsi] + or r12,r9 + or r12,r10 + or r12,r11 + mov rbp,-1 + + mov rax,QWORD PTR[rcx] + cmovnz r12,rbp + mov rsi,QWORD PTR[8+rcx] + mov rbx,QWORD PTR[16+rcx] + and rax,r12 + mov rbp,QWORD PTR[24+rcx] + and rsi,r12 + and rbx,r12 + and rbp,r12 + + sub rax,r8 + sbb rsi,r9 + sbb rbx,r10 + sbb rbp,r11 + + or rdx,rdx + + cmovz rax,r8 + cmovz rsi,r9 + mov QWORD PTR[rdi],rax + cmovz rbx,r10 + mov QWORD PTR[8+rdi],rsi + cmovz rbp,r11 + mov QWORD PTR[16+rdi],rbx + mov QWORD PTR[24+rdi],rbp + + mov r12,QWORD PTR[rsp] + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_cneg_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_cneg_mod_256:: +cneg_mod_256 ENDP + + +PUBLIC sub_mod_256 + + +ALIGN 32 +sub_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sub_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_sub_mod_256:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + + sub r8,QWORD PTR[rdx] + mov rax,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rdx] + mov rsi,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[24+rcx] + sbb rdx,rdx + + and rax,rdx + and rsi,rdx + and rbx,rdx + and rbp,rdx + + add r8,rax + adc r9,rsi + mov QWORD PTR[rdi],r8 + adc r10,rbx + mov QWORD PTR[8+rdi],r9 + adc r11,rbp + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_sub_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sub_mod_256:: +sub_mod_256 ENDP + + +PUBLIC check_mod_256 + + +ALIGN 32 +check_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_check_mod_256:: + mov rdi,rcx + mov rsi,rdx + + + + mov rax,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + + mov r8,rax + or rax,r9 + or rax,r10 + or rax,r11 + + sub r8,QWORD PTR[rsi] + sbb r9,QWORD PTR[8+rsi] + sbb r10,QWORD PTR[16+rsi] + sbb r11,QWORD PTR[24+rsi] + sbb rsi,rsi + + mov rdx,1 + cmp rax,0 + cmovne rax,rdx + and rax,rsi +$L$SEH_epilogue_check_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_check_mod_256:: +check_mod_256 ENDP + + +PUBLIC add_n_check_mod_256 + + +ALIGN 32 +add_n_check_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_add_n_check_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_add_n_check_mod_256:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + mov rax,r8 + adc r10,QWORD PTR[16+rdx] + mov rsi,r9 + adc r11,QWORD PTR[24+rdx] + sbb rdx,rdx + + mov rbx,r10 + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rcx] + mov rbp,r11 + sbb r11,QWORD PTR[24+rcx] + sbb rdx,0 + + cmovc r8,rax + cmovc r9,rsi + mov QWORD PTR[rdi],r8 + cmovc r10,rbx + mov QWORD PTR[8+rdi],r9 + cmovc r11,rbp + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + or r8,r9 + or r10,r11 + or r8,r10 + mov rax,1 + cmovz rax,r8 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_add_n_check_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_add_n_check_mod_256:: +add_n_check_mod_256 ENDP + + +PUBLIC sub_n_check_mod_256 + + +ALIGN 32 +sub_n_check_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sub_n_check_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_sub_n_check_mod_256:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + + sub r8,QWORD PTR[rdx] + mov rax,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rdx] + mov rsi,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[24+rcx] + sbb rdx,rdx + + and rax,rdx + and rsi,rdx + and rbx,rdx + and rbp,rdx + + add r8,rax + adc r9,rsi + mov QWORD PTR[rdi],r8 + adc r10,rbx + mov QWORD PTR[8+rdi],r9 + adc r11,rbp + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + or r8,r9 + or r10,r11 + or r8,r10 + mov rax,1 + cmovz rax,r8 + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_sub_n_check_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sub_n_check_mod_256:: +sub_n_check_mod_256 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_add_mod_256 + DD imagerel $L$SEH_body_add_mod_256 + DD imagerel $L$SEH_info_add_mod_256_prologue + + DD imagerel $L$SEH_body_add_mod_256 + DD imagerel $L$SEH_epilogue_add_mod_256 + DD imagerel $L$SEH_info_add_mod_256_body + + DD imagerel $L$SEH_epilogue_add_mod_256 + DD imagerel $L$SEH_end_add_mod_256 + DD imagerel $L$SEH_info_add_mod_256_epilogue + + DD imagerel $L$SEH_begin_mul_by_3_mod_256 + DD imagerel $L$SEH_body_mul_by_3_mod_256 + DD imagerel $L$SEH_info_mul_by_3_mod_256_prologue + + DD imagerel $L$SEH_body_mul_by_3_mod_256 + DD imagerel $L$SEH_epilogue_mul_by_3_mod_256 + DD imagerel $L$SEH_info_mul_by_3_mod_256_body + + DD imagerel $L$SEH_epilogue_mul_by_3_mod_256 + DD imagerel $L$SEH_end_mul_by_3_mod_256 + DD imagerel $L$SEH_info_mul_by_3_mod_256_epilogue + + DD imagerel $L$SEH_begin_lshift_mod_256 + DD imagerel $L$SEH_body_lshift_mod_256 + DD imagerel $L$SEH_info_lshift_mod_256_prologue + + DD imagerel $L$SEH_body_lshift_mod_256 + DD imagerel $L$SEH_epilogue_lshift_mod_256 + DD imagerel $L$SEH_info_lshift_mod_256_body + + DD imagerel $L$SEH_epilogue_lshift_mod_256 + DD imagerel $L$SEH_end_lshift_mod_256 + DD imagerel $L$SEH_info_lshift_mod_256_epilogue + + DD imagerel $L$SEH_begin_rshift_mod_256 + DD imagerel $L$SEH_body_rshift_mod_256 + DD imagerel $L$SEH_info_rshift_mod_256_prologue + + DD imagerel $L$SEH_body_rshift_mod_256 + DD imagerel $L$SEH_epilogue_rshift_mod_256 + DD imagerel $L$SEH_info_rshift_mod_256_body + + DD imagerel $L$SEH_epilogue_rshift_mod_256 + DD imagerel $L$SEH_end_rshift_mod_256 + DD imagerel $L$SEH_info_rshift_mod_256_epilogue + + DD imagerel $L$SEH_begin_cneg_mod_256 + DD imagerel $L$SEH_body_cneg_mod_256 + DD imagerel $L$SEH_info_cneg_mod_256_prologue + + DD imagerel $L$SEH_body_cneg_mod_256 + DD imagerel $L$SEH_epilogue_cneg_mod_256 + DD imagerel $L$SEH_info_cneg_mod_256_body + + DD imagerel $L$SEH_epilogue_cneg_mod_256 + DD imagerel $L$SEH_end_cneg_mod_256 + DD imagerel $L$SEH_info_cneg_mod_256_epilogue + + DD imagerel $L$SEH_begin_sub_mod_256 + DD imagerel $L$SEH_body_sub_mod_256 + DD imagerel $L$SEH_info_sub_mod_256_prologue + + DD imagerel $L$SEH_body_sub_mod_256 + DD imagerel $L$SEH_epilogue_sub_mod_256 + DD imagerel $L$SEH_info_sub_mod_256_body + + DD imagerel $L$SEH_epilogue_sub_mod_256 + DD imagerel $L$SEH_end_sub_mod_256 + DD imagerel $L$SEH_info_sub_mod_256_epilogue + + DD imagerel $L$SEH_epilogue_check_mod_256 + DD imagerel $L$SEH_end_check_mod_256 + DD imagerel $L$SEH_info_check_mod_256_epilogue + + DD imagerel $L$SEH_begin_add_n_check_mod_256 + DD imagerel $L$SEH_body_add_n_check_mod_256 + DD imagerel $L$SEH_info_add_n_check_mod_256_prologue + + DD imagerel $L$SEH_body_add_n_check_mod_256 + DD imagerel $L$SEH_epilogue_add_n_check_mod_256 + DD imagerel $L$SEH_info_add_n_check_mod_256_body + + DD imagerel $L$SEH_epilogue_add_n_check_mod_256 + DD imagerel $L$SEH_end_add_n_check_mod_256 + DD imagerel $L$SEH_info_add_n_check_mod_256_epilogue + + DD imagerel $L$SEH_begin_sub_n_check_mod_256 + DD imagerel $L$SEH_body_sub_n_check_mod_256 + DD imagerel $L$SEH_info_sub_n_check_mod_256_prologue + + DD imagerel $L$SEH_body_sub_n_check_mod_256 + DD imagerel $L$SEH_epilogue_sub_n_check_mod_256 + DD imagerel $L$SEH_info_sub_n_check_mod_256_body + + DD imagerel $L$SEH_epilogue_sub_n_check_mod_256 + DD imagerel $L$SEH_end_sub_n_check_mod_256 + DD imagerel $L$SEH_info_sub_n_check_mod_256_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_add_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_add_mod_256_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_add_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_3_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_3_mod_256_body:: +DB 1,0,11,0 +DB 000h,0c4h,000h,000h +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h,000h,000h,000h,000h +$L$SEH_info_mul_by_3_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_lshift_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_lshift_mod_256_body:: +DB 1,0,11,0 +DB 000h,0c4h,000h,000h +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h,000h,000h,000h,000h +$L$SEH_info_lshift_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_rshift_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_rshift_mod_256_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_rshift_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_cneg_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_cneg_mod_256_body:: +DB 1,0,11,0 +DB 000h,0c4h,000h,000h +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h,000h,000h,000h,000h +$L$SEH_info_cneg_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sub_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sub_mod_256_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_sub_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_check_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_add_n_check_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_add_n_check_mod_256_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_add_n_check_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sub_n_check_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sub_n_check_mod_256_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_sub_n_check_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/add_mod_384-armv8.asm b/crypto/blst_src/build/win64/add_mod_384-armv8.asm new file mode 100644 index 00000000000..4bf703a6da0 --- /dev/null +++ b/crypto/blst_src/build/win64/add_mod_384-armv8.asm @@ -0,0 +1,1001 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + + EXPORT |add_mod_384|[FUNC] + ALIGN 32 +|add_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__add_mod_384| PROC + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + +|__add_mod_384_ab_are_loaded| + adds x10,x10,x16 + adcs x11,x11,x17 + adcs x12,x12,x19 + adcs x13,x13,x20 + adcs x14,x14,x21 + adcs x15,x15,x22 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csello x10,x10,x16 + csello x11,x11,x17 + csello x12,x12,x19 + csello x13,x13,x20 + csello x14,x14,x21 + csello x15,x15,x22 + + ret + ENDP + + + + EXPORT |add_mod_384x|[FUNC] + ALIGN 32 +|add_mod_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __add_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __add_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |rshift_mod_384|[FUNC] + ALIGN 32 +|rshift_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +|$Loop_rshift_mod_384| + sub x2,x2,#1 + bl __rshift_mod_384 + cbnz x2,|$Loop_rshift_mod_384| + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__rshift_mod_384| PROC + sbfx x22,x10,#0,#1 + and x16,x22,x4 + and x17,x22,x5 + adds x10,x10,x16 + and x19,x22,x6 + adcs x11,x11,x17 + and x20,x22,x7 + adcs x12,x12,x19 + and x21,x22,x8 + adcs x13,x13,x20 + and x22,x22,x9 + adcs x14,x14,x21 + extr x10,x11,x10,#1 // a[0:5] >>= 1 + adcs x15,x15,x22 + extr x11,x12,x11,#1 + adc x22,xzr,xzr + extr x12,x13,x12,#1 + extr x13,x14,x13,#1 + extr x14,x15,x14,#1 + extr x15,x22,x15,#1 + ret + ENDP + + + + EXPORT |div_by_2_mod_384|[FUNC] + ALIGN 32 +|div_by_2_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __rshift_mod_384 + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |lshift_mod_384|[FUNC] + ALIGN 32 +|lshift_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + +|$Loop_lshift_mod_384| + sub x2,x2,#1 + bl __lshift_mod_384 + cbnz x2,|$Loop_lshift_mod_384| + + ldr x30,[sp,#8] + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__lshift_mod_384| PROC + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x16,x10,x4 + sbcs x17,x11,x5 + sbcs x19,x12,x6 + sbcs x20,x13,x7 + sbcs x21,x14,x8 + sbcs x22,x15,x9 + sbcs xzr,x3,xzr + + csello x10,x10,x16 + csello x11,x11,x17 + csello x12,x12,x19 + csello x13,x13,x20 + csello x14,x14,x21 + csello x15,x15,x22 + + ret + ENDP + + + + EXPORT |mul_by_3_mod_384|[FUNC] + ALIGN 32 +|mul_by_3_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |mul_by_8_mod_384|[FUNC] + ALIGN 32 +|mul_by_8_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |mul_by_3_mod_384x|[FUNC] + ALIGN 32 +|mul_by_3_mod_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + + bl __add_mod_384_ab_are_loaded + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + + ldp x16,x17,[x1,#48] + ldp x19,x20,[x1,#64] + ldp x21,x22,[x1,#80] + + bl __add_mod_384_ab_are_loaded + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |mul_by_8_mod_384x|[FUNC] + ALIGN 32 +|mul_by_8_mod_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + ldp x14,x15,[x1,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __lshift_mod_384 + bl __lshift_mod_384 + bl __lshift_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |cneg_mod_384|[FUNC] + ALIGN 32 +|cneg_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x10,x11,[x1] + ldp x4,x5,[x3] + ldp x12,x13,[x1,#16] + ldp x6,x7,[x3,#16] + + subs x16,x4,x10 + ldp x14,x15,[x1,#32] + ldp x8,x9,[x3,#32] + orr x3,x10,x11 + sbcs x17,x5,x11 + orr x3,x3,x12 + sbcs x19,x6,x12 + orr x3,x3,x13 + sbcs x20,x7,x13 + orr x3,x3,x14 + sbcs x21,x8,x14 + orr x3,x3,x15 + sbc x22,x9,x15 + + cmp x3,#0 + csetmne x3 + ands x2,x2,x3 + + cseleq x10,x10,x16 + cseleq x11,x11,x17 + cseleq x12,x12,x19 + cseleq x13,x13,x20 + stp x10,x11,[x0] + cseleq x14,x14,x21 + stp x12,x13,[x0,#16] + cseleq x15,x15,x22 + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sub_mod_384|[FUNC] + ALIGN 32 +|sub_mod_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + stp x14,x15,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__sub_mod_384| PROC + ldp x10,x11,[x1] + ldp x16,x17,[x2] + ldp x12,x13,[x1,#16] + ldp x19,x20,[x2,#16] + ldp x14,x15,[x1,#32] + ldp x21,x22,[x2,#32] + + subs x10,x10,x16 + sbcs x11,x11,x17 + sbcs x12,x12,x19 + sbcs x13,x13,x20 + sbcs x14,x14,x21 + sbcs x15,x15,x22 + sbc x3,xzr,xzr + + and x16,x4,x3 + and x17,x5,x3 + adds x10,x10,x16 + and x19,x6,x3 + adcs x11,x11,x17 + and x20,x7,x3 + adcs x12,x12,x19 + and x21,x8,x3 + adcs x13,x13,x20 + and x22,x9,x3 + adcs x14,x14,x21 + adc x15,x15,x22 + + ret + ENDP + + + + EXPORT |sub_mod_384x|[FUNC] + ALIGN 32 +|sub_mod_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x3] + ldp x6,x7,[x3,#16] + ldp x8,x9,[x3,#32] + + bl __sub_mod_384 + + stp x10,x11,[x0] + add x1,x1,#48 + stp x12,x13,[x0,#16] + add x2,x2,#48 + stp x14,x15,[x0,#32] + + bl __sub_mod_384 + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |mul_by_1_plus_i_mod_384x|[FUNC] + ALIGN 32 +|mul_by_1_plus_i_mod_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x4,x5,[x2] + ldp x6,x7,[x2,#16] + ldp x8,x9,[x2,#32] + add x2,x1,#48 + + bl __sub_mod_384 // a->re - a->im + + ldp x16,x17,[x1] + ldp x19,x20,[x1,#16] + ldp x21,x22,[x1,#32] + stp x10,x11,[x0] + ldp x10,x11,[x1,#48] + stp x12,x13,[x0,#16] + ldp x12,x13,[x1,#64] + stp x14,x15,[x0,#32] + ldp x14,x15,[x1,#80] + + bl __add_mod_384_ab_are_loaded // a->re + a->im + ldr x30,[sp,#8] + + stp x10,x11,[x0,#48] + stp x12,x13,[x0,#64] + stp x14,x15,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sgn0_pty_mod_384|[FUNC] + ALIGN 32 +|sgn0_pty_mod_384| PROC + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x0,x10,#1 + adds x10,x10,x10 + adcs x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x3,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x3,x3,xzr + + mvn x3,x3 + and x3,x3,#2 + orr x0,x0,x3 + + ret + ENDP + + + + EXPORT |sgn0_pty_mod_384x|[FUNC] + ALIGN 32 +|sgn0_pty_mod_384x| PROC + ldp x10,x11,[x0] + ldp x12,x13,[x0,#16] + ldp x14,x15,[x0,#32] + + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + ldp x8,x9,[x1,#32] + + and x2,x10,#1 + orr x3,x10,x11 + adds x10,x10,x10 + orr x3,x3,x12 + adcs x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + ldp x10,x11,[x0,#48] + ldp x12,x13,[x0,#64] + ldp x14,x15,[x0,#80] + + mvn x16,x16 + and x16,x16,#2 + orr x2,x2,x16 + + and x0,x10,#1 + orr x1,x10,x11 + adds x10,x10,x10 + orr x1,x1,x12 + adcs x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + adcs x15,x15,x15 + adc x16,xzr,xzr + + subs x10,x10,x4 + sbcs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbc x16,x16,xzr + + mvn x16,x16 + and x16,x16,#2 + orr x0,x0,x16 + + cmp x3,#0 + cseleq x3,x0,x2 + + cmp x1,#0 + cselne x1,x0,x2 + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ret + ENDP + + + EXPORT |vec_select_32|[FUNC] + ALIGN 32 +|vec_select_32| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + ENDP + + + EXPORT |vec_select_48|[FUNC] + ALIGN 32 +|vec_select_48| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + ENDP + + + EXPORT |vec_select_96|[FUNC] + ALIGN 32 +|vec_select_96| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + ENDP + + + EXPORT |vec_select_192|[FUNC] + ALIGN 32 +|vec_select_192| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + ENDP + + + EXPORT |vec_select_144|[FUNC] + ALIGN 32 +|vec_select_144| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + bit v1.16b, v4.16b, v6.16b + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0] + ret + ENDP + + + EXPORT |vec_select_288|[FUNC] + ALIGN 32 +|vec_select_288| PROC + dup v6.2d, x3 + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + cmeq v6.2d, v6.2d, #0 + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + bit v17.16b, v20.16b, v6.16b + ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0],#48 + bit v0.16b, v3.16b, v6.16b + ld1 {v16.2d, v17.2d, v18.2d}, [x1],#48 + bit v1.16b, v4.16b, v6.16b + ld1 {v19.2d, v20.2d, v21.2d}, [x2],#48 + bit v2.16b, v5.16b, v6.16b + st1 {v0.2d, v1.2d, v2.2d}, [x0],#48 + bit v16.16b, v19.16b, v6.16b + bit v17.16b, v20.16b, v6.16b + bit v18.16b, v21.16b, v6.16b + st1 {v16.2d, v17.2d, v18.2d}, [x0] + ret + ENDP + + + EXPORT |vec_prefetch|[FUNC] + ALIGN 32 +|vec_prefetch| PROC + add x1, x1, x0 + sub x1, x1, #1 + mov x2, #64 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + cselhi x2,xzr,x2 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + cselhi x2,xzr,x2 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + cselhi x2,xzr,x2 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + cselhi x2,xzr,x2 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + cselhi x2,xzr,x2 + prfm pldl1keep, [x0] + add x0, x0, x2 + cmp x0, x1 + cselhi x0,x1,x0 + prfm pldl1keep, [x0] + ret + ENDP + + + EXPORT |vec_is_zero_16x|[FUNC] + ALIGN 32 +|vec_is_zero_16x| PROC + ld1 {v0.2d}, [x0], #16 + lsr x1, x1, #4 + sub x1, x1, #1 + cbz x1, |$Loop_is_zero_done| + +|$Loop_is_zero| + ld1 {v1.2d}, [x0], #16 + orr v0.16b, v0.16b, v1.16b + sub x1, x1, #1 + cbnz x1, |$Loop_is_zero| + +|$Loop_is_zero_done| + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + cseleq x0,x0,xzr + ret + ENDP + + + EXPORT |vec_is_equal_16x|[FUNC] + ALIGN 32 +|vec_is_equal_16x| PROC + ld1 {v0.2d}, [x0], #16 + ld1 {v1.2d}, [x1], #16 + lsr x2, x2, #4 + eor v0.16b, v0.16b, v1.16b + +|$Loop_is_equal| + sub x2, x2, #1 + cbz x2, |$Loop_is_equal_done| + ld1 {v1.2d}, [x0], #16 + ld1 {v2.2d}, [x1], #16 + eor v1.16b, v1.16b, v2.16b + orr v0.16b, v0.16b, v1.16b + b |$Loop_is_equal| + nop + +|$Loop_is_equal_done| + dup v1.2d, v0.d[1] + orr v0.16b, v0.16b, v1.16b + mov x1, v0.d[0] + mov x0, #1 + cmp x1, #0 + cseleq x0,x0,xzr + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/add_mod_384-x86_64.asm b/crypto/blst_src/build/win64/add_mod_384-x86_64.asm new file mode 100644 index 00000000000..8a7b9e255db --- /dev/null +++ b/crypto/blst_src/build/win64/add_mod_384-x86_64.asm @@ -0,0 +1,2504 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC add_mod_384 + + +ALIGN 32 +add_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_add_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_add_mod_384:: + + + call __add_mod_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_add_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_add_mod_384:: +add_mod_384 ENDP + + +ALIGN 32 +__add_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + +__add_mod_384_a_is_loaded:: + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + adc r10,QWORD PTR[16+rdx] + mov r14,r8 + adc r11,QWORD PTR[24+rdx] + mov r15,r9 + adc r12,QWORD PTR[32+rdx] + mov rax,r10 + adc r13,QWORD PTR[40+rdx] + mov rbx,r11 + sbb rdx,rdx + + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + mov rbp,r12 + sbb r10,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rcx] + mov rsi,r13 + sbb r13,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + mov QWORD PTR[rdi],r8 + cmovc r11,rbx + mov QWORD PTR[8+rdi],r9 + cmovc r12,rbp + mov QWORD PTR[16+rdi],r10 + cmovc r13,rsi + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__add_mod_384 ENDP + +PUBLIC add_mod_384x + + +ALIGN 32 +add_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_add_mod_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,24 + +$L$SEH_body_add_mod_384x:: + + + mov QWORD PTR[rsp],rsi + mov QWORD PTR[8+rsp],rdx + lea rsi,QWORD PTR[48+rsi] + lea rdx,QWORD PTR[48+rdx] + lea rdi,QWORD PTR[48+rdi] + call __add_mod_384 + + mov rsi,QWORD PTR[rsp] + mov rdx,QWORD PTR[8+rsp] + lea rdi,QWORD PTR[((-48))+rdi] + call __add_mod_384 + + mov r15,QWORD PTR[((24+0))+rsp] + + mov r14,QWORD PTR[((24+8))+rsp] + + mov r13,QWORD PTR[((24+16))+rsp] + + mov r12,QWORD PTR[((24+24))+rsp] + + mov rbx,QWORD PTR[((24+32))+rsp] + + mov rbp,QWORD PTR[((24+40))+rsp] + + lea rsp,QWORD PTR[((24+48))+rsp] + +$L$SEH_epilogue_add_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_add_mod_384x:: +add_mod_384x ENDP + + +PUBLIC rshift_mod_384 + + +ALIGN 32 +rshift_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_rshift_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_rshift_mod_384:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + +$L$oop_rshift_mod_384:: + call __rshift_mod_384 + dec edx + jnz $L$oop_rshift_mod_384 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_rshift_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_rshift_mod_384:: +rshift_mod_384 ENDP + + +ALIGN 32 +__rshift_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov rsi,1 + mov r14,QWORD PTR[rcx] + and rsi,r8 + mov r15,QWORD PTR[8+rcx] + neg rsi + mov rax,QWORD PTR[16+rcx] + and r14,rsi + mov rbx,QWORD PTR[24+rcx] + and r15,rsi + mov rbp,QWORD PTR[32+rcx] + and rax,rsi + and rbx,rsi + and rbp,rsi + and rsi,QWORD PTR[40+rcx] + + add r14,r8 + adc r15,r9 + adc rax,r10 + adc rbx,r11 + adc rbp,r12 + adc rsi,r13 + sbb r13,r13 + + shr r14,1 + mov r8,r15 + shr r15,1 + mov r9,rax + shr rax,1 + mov r10,rbx + shr rbx,1 + mov r11,rbp + shr rbp,1 + mov r12,rsi + shr rsi,1 + shl r8,63 + shl r9,63 + or r8,r14 + shl r10,63 + or r9,r15 + shl r11,63 + or r10,rax + shl r12,63 + or r11,rbx + shl r13,63 + or r12,rbp + or r13,rsi + + DB 0F3h,0C3h ;repret +__rshift_mod_384 ENDP + +PUBLIC div_by_2_mod_384 + + +ALIGN 32 +div_by_2_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_div_by_2_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_div_by_2_mod_384:: + + + mov r8,QWORD PTR[rsi] + mov rcx,rdx + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + call __rshift_mod_384 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_div_by_2_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_div_by_2_mod_384:: +div_by_2_mod_384 ENDP + + +PUBLIC lshift_mod_384 + + +ALIGN 32 +lshift_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_lshift_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_lshift_mod_384:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + +$L$oop_lshift_mod_384:: + add r8,r8 + adc r9,r9 + adc r10,r10 + mov r14,r8 + adc r11,r11 + mov r15,r9 + adc r12,r12 + mov rax,r10 + adc r13,r13 + mov rbx,r11 + sbb rdi,rdi + + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + mov rbp,r12 + sbb r10,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rcx] + mov rsi,r13 + sbb r13,QWORD PTR[40+rcx] + sbb rdi,0 + + mov rdi,QWORD PTR[rsp] + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + cmovc r11,rbx + cmovc r12,rbp + cmovc r13,rsi + + dec edx + jnz $L$oop_lshift_mod_384 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_lshift_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_lshift_mod_384:: +lshift_mod_384 ENDP + + +ALIGN 32 +__lshift_mod_384 PROC PRIVATE + DB 243,15,30,250 + add r8,r8 + adc r9,r9 + adc r10,r10 + mov r14,r8 + adc r11,r11 + mov r15,r9 + adc r12,r12 + mov rax,r10 + adc r13,r13 + mov rbx,r11 + sbb rdx,rdx + + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + mov rbp,r12 + sbb r10,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rcx] + mov rsi,r13 + sbb r13,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + cmovc r11,rbx + cmovc r12,rbp + cmovc r13,rsi + + DB 0F3h,0C3h ;repret +__lshift_mod_384 ENDP + + +PUBLIC mul_by_3_mod_384 + + +ALIGN 32 +mul_by_3_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_3_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rsi + +$L$SEH_body_mul_by_3_mod_384:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov rcx,rdx + + call __lshift_mod_384 + + mov rdx,QWORD PTR[rsp] + call __add_mod_384_a_is_loaded + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mul_by_3_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_3_mod_384:: +mul_by_3_mod_384 ENDP + +PUBLIC mul_by_8_mod_384 + + +ALIGN 32 +mul_by_8_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_8_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_mul_by_8_mod_384:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov rcx,rdx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mul_by_8_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_8_mod_384:: +mul_by_8_mod_384 ENDP + + +PUBLIC mul_by_3_mod_384x + + +ALIGN 32 +mul_by_3_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_3_mod_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rsi + +$L$SEH_body_mul_by_3_mod_384x:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov rcx,rdx + + call __lshift_mod_384 + + mov rdx,QWORD PTR[rsp] + call __add_mod_384_a_is_loaded + + mov rsi,QWORD PTR[rsp] + lea rdi,QWORD PTR[48+rdi] + + mov r8,QWORD PTR[48+rsi] + mov r9,QWORD PTR[56+rsi] + mov r10,QWORD PTR[64+rsi] + mov r11,QWORD PTR[72+rsi] + mov r12,QWORD PTR[80+rsi] + mov r13,QWORD PTR[88+rsi] + + call __lshift_mod_384 + + mov rdx,8*6 + add rdx,QWORD PTR[rsp] + call __add_mod_384_a_is_loaded + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mul_by_3_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_3_mod_384x:: +mul_by_3_mod_384x ENDP + +PUBLIC mul_by_8_mod_384x + + +ALIGN 32 +mul_by_8_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_8_mod_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rsi + +$L$SEH_body_mul_by_8_mod_384x:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov rcx,rdx + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + mov rsi,QWORD PTR[rsp] + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + mov r8,QWORD PTR[((48+0))+rsi] + mov r9,QWORD PTR[((48+8))+rsi] + mov r10,QWORD PTR[((48+16))+rsi] + mov r11,QWORD PTR[((48+24))+rsi] + mov r12,QWORD PTR[((48+32))+rsi] + mov r13,QWORD PTR[((48+40))+rsi] + + call __lshift_mod_384 + call __lshift_mod_384 + call __lshift_mod_384 + + mov QWORD PTR[((48+0))+rdi],r8 + mov QWORD PTR[((48+8))+rdi],r9 + mov QWORD PTR[((48+16))+rdi],r10 + mov QWORD PTR[((48+24))+rdi],r11 + mov QWORD PTR[((48+32))+rdi],r12 + mov QWORD PTR[((48+40))+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mul_by_8_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_8_mod_384x:: +mul_by_8_mod_384x ENDP + + +PUBLIC cneg_mod_384 + + +ALIGN 32 +cneg_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_cneg_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdx + +$L$SEH_body_cneg_mod_384:: + + + mov rdx,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r8,rdx + mov r11,QWORD PTR[24+rsi] + or rdx,r9 + mov r12,QWORD PTR[32+rsi] + or rdx,r10 + mov r13,QWORD PTR[40+rsi] + or rdx,r11 + mov rsi,-1 + or rdx,r12 + or rdx,r13 + + mov r14,QWORD PTR[rcx] + cmovnz rdx,rsi + mov r15,QWORD PTR[8+rcx] + mov rax,QWORD PTR[16+rcx] + and r14,rdx + mov rbx,QWORD PTR[24+rcx] + and r15,rdx + mov rbp,QWORD PTR[32+rcx] + and rax,rdx + mov rsi,QWORD PTR[40+rcx] + and rbx,rdx + mov rcx,QWORD PTR[rsp] + and rbp,rdx + and rsi,rdx + + sub r14,r8 + sbb r15,r9 + sbb rax,r10 + sbb rbx,r11 + sbb rbp,r12 + sbb rsi,r13 + + or rcx,rcx + + cmovz r14,r8 + cmovz r15,r9 + cmovz rax,r10 + mov QWORD PTR[rdi],r14 + cmovz rbx,r11 + mov QWORD PTR[8+rdi],r15 + cmovz rbp,r12 + mov QWORD PTR[16+rdi],rax + cmovz rsi,r13 + mov QWORD PTR[24+rdi],rbx + mov QWORD PTR[32+rdi],rbp + mov QWORD PTR[40+rdi],rsi + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_cneg_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_cneg_mod_384:: +cneg_mod_384 ENDP + + +PUBLIC sub_mod_384 + + +ALIGN 32 +sub_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sub_mod_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sub_mod_384:: + + + call __sub_mod_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sub_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sub_mod_384:: +sub_mod_384 ENDP + + +ALIGN 32 +__sub_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + sub r8,QWORD PTR[rdx] + mov r14,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rdx] + mov r15,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rdx] + mov rax,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rdx] + mov rbx,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rdx] + mov rbp,QWORD PTR[32+rcx] + sbb r13,QWORD PTR[40+rdx] + mov rsi,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r14,rdx + and r15,rdx + and rax,rdx + and rbx,rdx + and rbp,rdx + and rsi,rdx + + add r8,r14 + adc r9,r15 + mov QWORD PTR[rdi],r8 + adc r10,rax + mov QWORD PTR[8+rdi],r9 + adc r11,rbx + mov QWORD PTR[16+rdi],r10 + adc r12,rbp + mov QWORD PTR[24+rdi],r11 + adc r13,rsi + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__sub_mod_384 ENDP + +PUBLIC sub_mod_384x + + +ALIGN 32 +sub_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sub_mod_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,24 + +$L$SEH_body_sub_mod_384x:: + + + mov QWORD PTR[rsp],rsi + mov QWORD PTR[8+rsp],rdx + lea rsi,QWORD PTR[48+rsi] + lea rdx,QWORD PTR[48+rdx] + lea rdi,QWORD PTR[48+rdi] + call __sub_mod_384 + + mov rsi,QWORD PTR[rsp] + mov rdx,QWORD PTR[8+rsp] + lea rdi,QWORD PTR[((-48))+rdi] + call __sub_mod_384 + + mov r15,QWORD PTR[((24+0))+rsp] + + mov r14,QWORD PTR[((24+8))+rsp] + + mov r13,QWORD PTR[((24+16))+rsp] + + mov r12,QWORD PTR[((24+24))+rsp] + + mov rbx,QWORD PTR[((24+32))+rsp] + + mov rbp,QWORD PTR[((24+40))+rsp] + + lea rsp,QWORD PTR[((24+48))+rsp] + +$L$SEH_epilogue_sub_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sub_mod_384x:: +sub_mod_384x ENDP +PUBLIC mul_by_1_plus_i_mod_384x + + +ALIGN 32 +mul_by_1_plus_i_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_by_1_plus_i_mod_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,56 + +$L$SEH_body_mul_by_1_plus_i_mod_384x:: + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,r8 + add r8,QWORD PTR[48+rsi] + mov r15,r9 + adc r9,QWORD PTR[56+rsi] + mov rax,r10 + adc r10,QWORD PTR[64+rsi] + mov rbx,r11 + adc r11,QWORD PTR[72+rsi] + mov rcx,r12 + adc r12,QWORD PTR[80+rsi] + mov rbp,r13 + adc r13,QWORD PTR[88+rsi] + mov QWORD PTR[48+rsp],rdi + sbb rdi,rdi + + sub r14,QWORD PTR[48+rsi] + sbb r15,QWORD PTR[56+rsi] + sbb rax,QWORD PTR[64+rsi] + sbb rbx,QWORD PTR[72+rsi] + sbb rcx,QWORD PTR[80+rsi] + sbb rbp,QWORD PTR[88+rsi] + sbb rsi,rsi + + mov QWORD PTR[rsp],r8 + mov r8,QWORD PTR[rdx] + mov QWORD PTR[8+rsp],r9 + mov r9,QWORD PTR[8+rdx] + mov QWORD PTR[16+rsp],r10 + mov r10,QWORD PTR[16+rdx] + mov QWORD PTR[24+rsp],r11 + mov r11,QWORD PTR[24+rdx] + mov QWORD PTR[32+rsp],r12 + and r8,rsi + mov r12,QWORD PTR[32+rdx] + mov QWORD PTR[40+rsp],r13 + and r9,rsi + mov r13,QWORD PTR[40+rdx] + and r10,rsi + and r11,rsi + and r12,rsi + and r13,rsi + mov rsi,QWORD PTR[48+rsp] + + add r14,r8 + mov r8,QWORD PTR[rsp] + adc r15,r9 + mov r9,QWORD PTR[8+rsp] + adc rax,r10 + mov r10,QWORD PTR[16+rsp] + adc rbx,r11 + mov r11,QWORD PTR[24+rsp] + adc rcx,r12 + mov r12,QWORD PTR[32+rsp] + adc rbp,r13 + mov r13,QWORD PTR[40+rsp] + + mov QWORD PTR[rsi],r14 + mov r14,r8 + mov QWORD PTR[8+rsi],r15 + mov QWORD PTR[16+rsi],rax + mov r15,r9 + mov QWORD PTR[24+rsi],rbx + mov QWORD PTR[32+rsi],rcx + mov rax,r10 + mov QWORD PTR[40+rsi],rbp + + sub r8,QWORD PTR[rdx] + mov rbx,r11 + sbb r9,QWORD PTR[8+rdx] + sbb r10,QWORD PTR[16+rdx] + mov rcx,r12 + sbb r11,QWORD PTR[24+rdx] + sbb r12,QWORD PTR[32+rdx] + mov rbp,r13 + sbb r13,QWORD PTR[40+rdx] + sbb rdi,0 + + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + mov QWORD PTR[48+rsi],r8 + cmovc r11,rbx + mov QWORD PTR[56+rsi],r9 + cmovc r12,rcx + mov QWORD PTR[64+rsi],r10 + cmovc r13,rbp + mov QWORD PTR[72+rsi],r11 + mov QWORD PTR[80+rsi],r12 + mov QWORD PTR[88+rsi],r13 + + mov r15,QWORD PTR[((56+0))+rsp] + + mov r14,QWORD PTR[((56+8))+rsp] + + mov r13,QWORD PTR[((56+16))+rsp] + + mov r12,QWORD PTR[((56+24))+rsp] + + mov rbx,QWORD PTR[((56+32))+rsp] + + mov rbp,QWORD PTR[((56+40))+rsp] + + lea rsp,QWORD PTR[((56+48))+rsp] + +$L$SEH_epilogue_mul_by_1_plus_i_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_by_1_plus_i_mod_384x:: +mul_by_1_plus_i_mod_384x ENDP +PUBLIC sgn0_pty_mod_384 + + +ALIGN 32 +sgn0_pty_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0_pty_mod_384:: + mov rdi,rcx + mov rsi,rdx + + + +$L$SEH_body_sgn0_pty_mod_384:: + + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov rcx,QWORD PTR[32+rdi] + mov rdx,QWORD PTR[40+rdi] + + xor rax,rax + mov rdi,r8 + add r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rcx,rcx + adc rdx,rdx + adc rax,0 + + sub r8,QWORD PTR[rsi] + sbb r9,QWORD PTR[8+rsi] + sbb r10,QWORD PTR[16+rsi] + sbb r11,QWORD PTR[24+rsi] + sbb rcx,QWORD PTR[32+rsi] + sbb rdx,QWORD PTR[40+rsi] + sbb rax,0 + + not rax + and rdi,1 + and rax,2 + or rax,rdi + +$L$SEH_epilogue_sgn0_pty_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0_pty_mod_384:: +sgn0_pty_mod_384 ENDP + +PUBLIC sgn0_pty_mod_384x + + +ALIGN 32 +sgn0_pty_mod_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0_pty_mod_384x:: + mov rdi,rcx + mov rsi,rdx + + + + push rbp + + push rbx + + sub rsp,8 + +$L$SEH_body_sgn0_pty_mod_384x:: + + + mov r8,QWORD PTR[48+rdi] + mov r9,QWORD PTR[56+rdi] + mov r10,QWORD PTR[64+rdi] + mov r11,QWORD PTR[72+rdi] + mov rcx,QWORD PTR[80+rdi] + mov rdx,QWORD PTR[88+rdi] + + mov rbx,r8 + or r8,r9 + or r8,r10 + or r8,r11 + or r8,rcx + or r8,rdx + + lea rax,QWORD PTR[rdi] + xor rdi,rdi + mov rbp,rbx + add rbx,rbx + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rcx,rcx + adc rdx,rdx + adc rdi,0 + + sub rbx,QWORD PTR[rsi] + sbb r9,QWORD PTR[8+rsi] + sbb r10,QWORD PTR[16+rsi] + sbb r11,QWORD PTR[24+rsi] + sbb rcx,QWORD PTR[32+rsi] + sbb rdx,QWORD PTR[40+rsi] + sbb rdi,0 + + mov QWORD PTR[rsp],r8 + not rdi + and rbp,1 + and rdi,2 + or rdi,rbp + + mov r8,QWORD PTR[rax] + mov r9,QWORD PTR[8+rax] + mov r10,QWORD PTR[16+rax] + mov r11,QWORD PTR[24+rax] + mov rcx,QWORD PTR[32+rax] + mov rdx,QWORD PTR[40+rax] + + mov rbx,r8 + or r8,r9 + or r8,r10 + or r8,r11 + or r8,rcx + or r8,rdx + + xor rax,rax + mov rbp,rbx + add rbx,rbx + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rcx,rcx + adc rdx,rdx + adc rax,0 + + sub rbx,QWORD PTR[rsi] + sbb r9,QWORD PTR[8+rsi] + sbb r10,QWORD PTR[16+rsi] + sbb r11,QWORD PTR[24+rsi] + sbb rcx,QWORD PTR[32+rsi] + sbb rdx,QWORD PTR[40+rsi] + sbb rax,0 + + mov rbx,QWORD PTR[rsp] + + not rax + + test r8,r8 + cmovz rbp,rdi + + test rbx,rbx + cmovnz rax,rdi + + and rbp,1 + and rax,2 + or rax,rbp + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_sgn0_pty_mod_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0_pty_mod_384x:: +sgn0_pty_mod_384x ENDP +PUBLIC vec_select_32 + + +ALIGN 32 +vec_select_32 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[16+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[16+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[16+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-16))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-16))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-16)+rcx],xmm0 + pand xmm2,xmm4 + pand xmm3,xmm5 + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-16)+rcx],xmm2 + DB 0F3h,0C3h ;repret +vec_select_32 ENDP +PUBLIC vec_select_48 + + +ALIGN 32 +vec_select_48 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[24+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[24+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[24+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-24))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-24))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-24)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((16+16-24))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((16+16-24))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-24)+rcx],xmm2 + pand xmm0,xmm4 + pand xmm1,xmm5 + por xmm0,xmm1 + movdqu XMMWORD PTR[(32-24)+rcx],xmm0 + DB 0F3h,0C3h ;repret +vec_select_48 ENDP +PUBLIC vec_select_96 + + +ALIGN 32 +vec_select_96 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[48+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[48+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[48+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-48))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-48))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-48)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((16+16-48))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((16+16-48))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-48)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((32+16-48))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((32+16-48))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(32-48)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((48+16-48))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((48+16-48))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(48-48)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((64+16-48))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((64+16-48))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(64-48)+rcx],xmm0 + pand xmm2,xmm4 + pand xmm3,xmm5 + por xmm2,xmm3 + movdqu XMMWORD PTR[(80-48)+rcx],xmm2 + DB 0F3h,0C3h ;repret +vec_select_96 ENDP +PUBLIC vec_select_192 + + +ALIGN 32 +vec_select_192 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[96+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[96+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[96+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-96)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((16+16-96))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((16+16-96))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-96)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((32+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((32+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(32-96)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((48+16-96))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((48+16-96))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(48-96)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((64+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((64+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(64-96)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((80+16-96))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((80+16-96))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(80-96)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((96+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((96+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(96-96)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((112+16-96))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((112+16-96))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(112-96)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((128+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((128+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(128-96)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((144+16-96))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((144+16-96))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(144-96)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((160+16-96))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((160+16-96))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(160-96)+rcx],xmm0 + pand xmm2,xmm4 + pand xmm3,xmm5 + por xmm2,xmm3 + movdqu XMMWORD PTR[(176-96)+rcx],xmm2 + DB 0F3h,0C3h ;repret +vec_select_192 ENDP +PUBLIC vec_select_144 + + +ALIGN 32 +vec_select_144 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[72+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[72+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[72+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-72))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-72))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-72)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((16+16-72))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((16+16-72))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-72)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((32+16-72))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((32+16-72))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(32-72)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((48+16-72))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((48+16-72))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(48-72)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((64+16-72))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((64+16-72))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(64-72)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((80+16-72))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((80+16-72))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(80-72)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((96+16-72))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((96+16-72))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(96-72)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((112+16-72))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((112+16-72))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(112-72)+rcx],xmm2 + pand xmm0,xmm4 + pand xmm1,xmm5 + por xmm0,xmm1 + movdqu XMMWORD PTR[(128-72)+rcx],xmm0 + DB 0F3h,0C3h ;repret +vec_select_144 ENDP +PUBLIC vec_select_288 + + +ALIGN 32 +vec_select_288 PROC PUBLIC + DB 243,15,30,250 + movd xmm5,r9d + pxor xmm4,xmm4 + pshufd xmm5,xmm5,0 + movdqu xmm0,XMMWORD PTR[rdx] + lea rdx,QWORD PTR[144+rdx] + pcmpeqd xmm5,xmm4 + movdqu xmm1,XMMWORD PTR[r8] + lea r8,QWORD PTR[144+r8] + pcmpeqd xmm4,xmm5 + lea rcx,QWORD PTR[144+rcx] + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((0+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((0+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(0-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((16+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((16+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(16-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((32+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((32+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(32-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((48+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((48+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(48-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((64+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((64+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(64-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((80+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((80+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(80-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((96+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((96+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(96-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((112+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((112+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(112-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((128+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((128+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(128-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((144+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((144+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(144-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((160+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((160+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(160-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((176+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((176+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(176-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((192+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((192+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(192-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((208+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((208+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(208-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((224+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((224+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(224-144)+rcx],xmm0 + pand xmm2,xmm4 + movdqu xmm0,XMMWORD PTR[((240+16-144))+rdx] + pand xmm3,xmm5 + movdqu xmm1,XMMWORD PTR[((240+16-144))+r8] + por xmm2,xmm3 + movdqu XMMWORD PTR[(240-144)+rcx],xmm2 + pand xmm0,xmm4 + movdqu xmm2,XMMWORD PTR[((256+16-144))+rdx] + pand xmm1,xmm5 + movdqu xmm3,XMMWORD PTR[((256+16-144))+r8] + por xmm0,xmm1 + movdqu XMMWORD PTR[(256-144)+rcx],xmm0 + pand xmm2,xmm4 + pand xmm3,xmm5 + por xmm2,xmm3 + movdqu XMMWORD PTR[(272-144)+rcx],xmm2 + DB 0F3h,0C3h ;repret +vec_select_288 ENDP +PUBLIC vec_prefetch + + +ALIGN 32 +vec_prefetch PROC PUBLIC + DB 243,15,30,250 + lea rdx,QWORD PTR[((-1))+rdx*1+rcx] + mov rax,64 + xor r8,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + cmova rax,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + cmova rax,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + cmova rax,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + cmova rax,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + cmova rax,r8 + prefetchnta [rcx] + lea rcx,QWORD PTR[rax*1+rcx] + cmp rcx,rdx + cmova rcx,rdx + prefetchnta [rcx] + DB 0F3h,0C3h ;repret +vec_prefetch ENDP +PUBLIC vec_is_zero_16x + + +ALIGN 32 +vec_is_zero_16x PROC PUBLIC + DB 243,15,30,250 + shr edx,4 + movdqu xmm0,XMMWORD PTR[rcx] + lea rcx,QWORD PTR[16+rcx] + +$L$oop_is_zero:: + dec edx + jz $L$oop_is_zero_done + movdqu xmm1,XMMWORD PTR[rcx] + lea rcx,QWORD PTR[16+rcx] + por xmm0,xmm1 + jmp $L$oop_is_zero + +$L$oop_is_zero_done:: + pshufd xmm1,xmm0,04eh + por xmm0,xmm1 +DB 102,72,15,126,192 + inc edx + test rax,rax + cmovnz eax,edx + xor eax,1 + DB 0F3h,0C3h ;repret +vec_is_zero_16x ENDP +PUBLIC vec_is_equal_16x + + +ALIGN 32 +vec_is_equal_16x PROC PUBLIC + DB 243,15,30,250 + shr r8d,4 + movdqu xmm0,XMMWORD PTR[rcx] + movdqu xmm1,XMMWORD PTR[rdx] + sub rdx,rcx + lea rcx,QWORD PTR[16+rcx] + pxor xmm0,xmm1 + +$L$oop_is_equal:: + dec r8d + jz $L$oop_is_equal_done + movdqu xmm1,XMMWORD PTR[rcx] + movdqu xmm2,XMMWORD PTR[rdx*1+rcx] + lea rcx,QWORD PTR[16+rcx] + pxor xmm1,xmm2 + por xmm0,xmm1 + jmp $L$oop_is_equal + +$L$oop_is_equal_done:: + pshufd xmm1,xmm0,04eh + por xmm0,xmm1 +DB 102,72,15,126,192 + inc r8d + test rax,rax + cmovnz eax,r8d + xor eax,1 + DB 0F3h,0C3h ;repret +vec_is_equal_16x ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_add_mod_384 + DD imagerel $L$SEH_body_add_mod_384 + DD imagerel $L$SEH_info_add_mod_384_prologue + + DD imagerel $L$SEH_body_add_mod_384 + DD imagerel $L$SEH_epilogue_add_mod_384 + DD imagerel $L$SEH_info_add_mod_384_body + + DD imagerel $L$SEH_epilogue_add_mod_384 + DD imagerel $L$SEH_end_add_mod_384 + DD imagerel $L$SEH_info_add_mod_384_epilogue + + DD imagerel $L$SEH_begin_add_mod_384x + DD imagerel $L$SEH_body_add_mod_384x + DD imagerel $L$SEH_info_add_mod_384x_prologue + + DD imagerel $L$SEH_body_add_mod_384x + DD imagerel $L$SEH_epilogue_add_mod_384x + DD imagerel $L$SEH_info_add_mod_384x_body + + DD imagerel $L$SEH_epilogue_add_mod_384x + DD imagerel $L$SEH_end_add_mod_384x + DD imagerel $L$SEH_info_add_mod_384x_epilogue + + DD imagerel $L$SEH_begin_rshift_mod_384 + DD imagerel $L$SEH_body_rshift_mod_384 + DD imagerel $L$SEH_info_rshift_mod_384_prologue + + DD imagerel $L$SEH_body_rshift_mod_384 + DD imagerel $L$SEH_epilogue_rshift_mod_384 + DD imagerel $L$SEH_info_rshift_mod_384_body + + DD imagerel $L$SEH_epilogue_rshift_mod_384 + DD imagerel $L$SEH_end_rshift_mod_384 + DD imagerel $L$SEH_info_rshift_mod_384_epilogue + + DD imagerel $L$SEH_begin_div_by_2_mod_384 + DD imagerel $L$SEH_body_div_by_2_mod_384 + DD imagerel $L$SEH_info_div_by_2_mod_384_prologue + + DD imagerel $L$SEH_body_div_by_2_mod_384 + DD imagerel $L$SEH_epilogue_div_by_2_mod_384 + DD imagerel $L$SEH_info_div_by_2_mod_384_body + + DD imagerel $L$SEH_epilogue_div_by_2_mod_384 + DD imagerel $L$SEH_end_div_by_2_mod_384 + DD imagerel $L$SEH_info_div_by_2_mod_384_epilogue + + DD imagerel $L$SEH_begin_lshift_mod_384 + DD imagerel $L$SEH_body_lshift_mod_384 + DD imagerel $L$SEH_info_lshift_mod_384_prologue + + DD imagerel $L$SEH_body_lshift_mod_384 + DD imagerel $L$SEH_epilogue_lshift_mod_384 + DD imagerel $L$SEH_info_lshift_mod_384_body + + DD imagerel $L$SEH_epilogue_lshift_mod_384 + DD imagerel $L$SEH_end_lshift_mod_384 + DD imagerel $L$SEH_info_lshift_mod_384_epilogue + + DD imagerel $L$SEH_begin_mul_by_3_mod_384 + DD imagerel $L$SEH_body_mul_by_3_mod_384 + DD imagerel $L$SEH_info_mul_by_3_mod_384_prologue + + DD imagerel $L$SEH_body_mul_by_3_mod_384 + DD imagerel $L$SEH_epilogue_mul_by_3_mod_384 + DD imagerel $L$SEH_info_mul_by_3_mod_384_body + + DD imagerel $L$SEH_epilogue_mul_by_3_mod_384 + DD imagerel $L$SEH_end_mul_by_3_mod_384 + DD imagerel $L$SEH_info_mul_by_3_mod_384_epilogue + + DD imagerel $L$SEH_begin_mul_by_8_mod_384 + DD imagerel $L$SEH_body_mul_by_8_mod_384 + DD imagerel $L$SEH_info_mul_by_8_mod_384_prologue + + DD imagerel $L$SEH_body_mul_by_8_mod_384 + DD imagerel $L$SEH_epilogue_mul_by_8_mod_384 + DD imagerel $L$SEH_info_mul_by_8_mod_384_body + + DD imagerel $L$SEH_epilogue_mul_by_8_mod_384 + DD imagerel $L$SEH_end_mul_by_8_mod_384 + DD imagerel $L$SEH_info_mul_by_8_mod_384_epilogue + + DD imagerel $L$SEH_begin_mul_by_3_mod_384x + DD imagerel $L$SEH_body_mul_by_3_mod_384x + DD imagerel $L$SEH_info_mul_by_3_mod_384x_prologue + + DD imagerel $L$SEH_body_mul_by_3_mod_384x + DD imagerel $L$SEH_epilogue_mul_by_3_mod_384x + DD imagerel $L$SEH_info_mul_by_3_mod_384x_body + + DD imagerel $L$SEH_epilogue_mul_by_3_mod_384x + DD imagerel $L$SEH_end_mul_by_3_mod_384x + DD imagerel $L$SEH_info_mul_by_3_mod_384x_epilogue + + DD imagerel $L$SEH_begin_mul_by_8_mod_384x + DD imagerel $L$SEH_body_mul_by_8_mod_384x + DD imagerel $L$SEH_info_mul_by_8_mod_384x_prologue + + DD imagerel $L$SEH_body_mul_by_8_mod_384x + DD imagerel $L$SEH_epilogue_mul_by_8_mod_384x + DD imagerel $L$SEH_info_mul_by_8_mod_384x_body + + DD imagerel $L$SEH_epilogue_mul_by_8_mod_384x + DD imagerel $L$SEH_end_mul_by_8_mod_384x + DD imagerel $L$SEH_info_mul_by_8_mod_384x_epilogue + + DD imagerel $L$SEH_begin_cneg_mod_384 + DD imagerel $L$SEH_body_cneg_mod_384 + DD imagerel $L$SEH_info_cneg_mod_384_prologue + + DD imagerel $L$SEH_body_cneg_mod_384 + DD imagerel $L$SEH_epilogue_cneg_mod_384 + DD imagerel $L$SEH_info_cneg_mod_384_body + + DD imagerel $L$SEH_epilogue_cneg_mod_384 + DD imagerel $L$SEH_end_cneg_mod_384 + DD imagerel $L$SEH_info_cneg_mod_384_epilogue + + DD imagerel $L$SEH_begin_sub_mod_384 + DD imagerel $L$SEH_body_sub_mod_384 + DD imagerel $L$SEH_info_sub_mod_384_prologue + + DD imagerel $L$SEH_body_sub_mod_384 + DD imagerel $L$SEH_epilogue_sub_mod_384 + DD imagerel $L$SEH_info_sub_mod_384_body + + DD imagerel $L$SEH_epilogue_sub_mod_384 + DD imagerel $L$SEH_end_sub_mod_384 + DD imagerel $L$SEH_info_sub_mod_384_epilogue + + DD imagerel $L$SEH_begin_sub_mod_384x + DD imagerel $L$SEH_body_sub_mod_384x + DD imagerel $L$SEH_info_sub_mod_384x_prologue + + DD imagerel $L$SEH_body_sub_mod_384x + DD imagerel $L$SEH_epilogue_sub_mod_384x + DD imagerel $L$SEH_info_sub_mod_384x_body + + DD imagerel $L$SEH_epilogue_sub_mod_384x + DD imagerel $L$SEH_end_sub_mod_384x + DD imagerel $L$SEH_info_sub_mod_384x_epilogue + + DD imagerel $L$SEH_begin_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_body_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_info_mul_by_1_plus_i_mod_384x_prologue + + DD imagerel $L$SEH_body_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_epilogue_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_info_mul_by_1_plus_i_mod_384x_body + + DD imagerel $L$SEH_epilogue_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_end_mul_by_1_plus_i_mod_384x + DD imagerel $L$SEH_info_mul_by_1_plus_i_mod_384x_epilogue + + DD imagerel $L$SEH_begin_sgn0_pty_mod_384 + DD imagerel $L$SEH_body_sgn0_pty_mod_384 + DD imagerel $L$SEH_info_sgn0_pty_mod_384_prologue + + DD imagerel $L$SEH_body_sgn0_pty_mod_384 + DD imagerel $L$SEH_epilogue_sgn0_pty_mod_384 + DD imagerel $L$SEH_info_sgn0_pty_mod_384_body + + DD imagerel $L$SEH_epilogue_sgn0_pty_mod_384 + DD imagerel $L$SEH_end_sgn0_pty_mod_384 + DD imagerel $L$SEH_info_sgn0_pty_mod_384_epilogue + + DD imagerel $L$SEH_begin_sgn0_pty_mod_384x + DD imagerel $L$SEH_body_sgn0_pty_mod_384x + DD imagerel $L$SEH_info_sgn0_pty_mod_384x_prologue + + DD imagerel $L$SEH_body_sgn0_pty_mod_384x + DD imagerel $L$SEH_epilogue_sgn0_pty_mod_384x + DD imagerel $L$SEH_info_sgn0_pty_mod_384x_body + + DD imagerel $L$SEH_epilogue_sgn0_pty_mod_384x + DD imagerel $L$SEH_end_sgn0_pty_mod_384x + DD imagerel $L$SEH_info_sgn0_pty_mod_384x_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_add_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_add_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_add_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_add_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_add_mod_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,003h,000h +DB 000h,0e4h,004h,000h +DB 000h,0d4h,005h,000h +DB 000h,0c4h,006h,000h +DB 000h,034h,007h,000h +DB 000h,054h,008h,000h +DB 000h,074h,00ah,000h +DB 000h,064h,00bh,000h +DB 000h,082h +DB 000h,000h +$L$SEH_info_add_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_rshift_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_rshift_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_rshift_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_div_by_2_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_div_by_2_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_div_by_2_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_lshift_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_lshift_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_lshift_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_3_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_3_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mul_by_3_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_8_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_8_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mul_by_8_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_3_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_3_mod_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mul_by_3_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_8_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_8_mod_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mul_by_8_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_cneg_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_cneg_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_cneg_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sub_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sub_mod_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sub_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sub_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sub_mod_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,003h,000h +DB 000h,0e4h,004h,000h +DB 000h,0d4h,005h,000h +DB 000h,0c4h,006h,000h +DB 000h,034h,007h,000h +DB 000h,054h,008h,000h +DB 000h,074h,00ah,000h +DB 000h,064h,00bh,000h +DB 000h,082h +DB 000h,000h +$L$SEH_info_sub_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_by_1_plus_i_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_by_1_plus_i_mod_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,007h,000h +DB 000h,0e4h,008h,000h +DB 000h,0d4h,009h,000h +DB 000h,0c4h,00ah,000h +DB 000h,034h,00bh,000h +DB 000h,054h,00ch,000h +DB 000h,074h,00eh,000h +DB 000h,064h,00fh,000h +DB 000h,0c2h +DB 000h,000h +$L$SEH_info_mul_by_1_plus_i_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0_pty_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0_pty_mod_384_body:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h +$L$SEH_info_sgn0_pty_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0_pty_mod_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0_pty_mod_384x_body:: +DB 1,0,9,0 +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h +$L$SEH_info_sgn0_pty_mod_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/add_mod_384x384-x86_64.asm b/crypto/blst_src/build/win64/add_mod_384x384-x86_64.asm new file mode 100644 index 00000000000..57d1752fd3c --- /dev/null +++ b/crypto/blst_src/build/win64/add_mod_384x384-x86_64.asm @@ -0,0 +1,334 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + + +ALIGN 32 +__add_mod_384x384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + + add r8,QWORD PTR[rdx] + mov r15,QWORD PTR[56+rsi] + adc r9,QWORD PTR[8+rdx] + mov rax,QWORD PTR[64+rsi] + adc r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[72+rsi] + adc r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[80+rsi] + adc r12,QWORD PTR[32+rdx] + mov rsi,QWORD PTR[88+rsi] + adc r13,QWORD PTR[40+rdx] + mov QWORD PTR[rdi],r8 + adc r14,QWORD PTR[48+rdx] + mov QWORD PTR[8+rdi],r9 + adc r15,QWORD PTR[56+rdx] + mov QWORD PTR[16+rdi],r10 + adc rax,QWORD PTR[64+rdx] + mov QWORD PTR[32+rdi],r12 + mov r8,r14 + adc rbx,QWORD PTR[72+rdx] + mov QWORD PTR[24+rdi],r11 + mov r9,r15 + adc rbp,QWORD PTR[80+rdx] + mov QWORD PTR[40+rdi],r13 + mov r10,rax + adc rsi,QWORD PTR[88+rdx] + mov r11,rbx + sbb rdx,rdx + + sub r14,QWORD PTR[rcx] + sbb r15,QWORD PTR[8+rcx] + mov r12,rbp + sbb rax,QWORD PTR[16+rcx] + sbb rbx,QWORD PTR[24+rcx] + sbb rbp,QWORD PTR[32+rcx] + mov r13,rsi + sbb rsi,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r14,r8 + cmovc r15,r9 + cmovc rax,r10 + mov QWORD PTR[48+rdi],r14 + cmovc rbx,r11 + mov QWORD PTR[56+rdi],r15 + cmovc rbp,r12 + mov QWORD PTR[64+rdi],rax + cmovc rsi,r13 + mov QWORD PTR[72+rdi],rbx + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rsi + + DB 0F3h,0C3h ;repret +__add_mod_384x384 ENDP + + +ALIGN 32 +__sub_mod_384x384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + + sub r8,QWORD PTR[rdx] + mov r15,QWORD PTR[56+rsi] + sbb r9,QWORD PTR[8+rdx] + mov rax,QWORD PTR[64+rsi] + sbb r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[72+rsi] + sbb r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[80+rsi] + sbb r12,QWORD PTR[32+rdx] + mov rsi,QWORD PTR[88+rsi] + sbb r13,QWORD PTR[40+rdx] + mov QWORD PTR[rdi],r8 + sbb r14,QWORD PTR[48+rdx] + mov r8,QWORD PTR[rcx] + mov QWORD PTR[8+rdi],r9 + sbb r15,QWORD PTR[56+rdx] + mov r9,QWORD PTR[8+rcx] + mov QWORD PTR[16+rdi],r10 + sbb rax,QWORD PTR[64+rdx] + mov r10,QWORD PTR[16+rcx] + mov QWORD PTR[24+rdi],r11 + sbb rbx,QWORD PTR[72+rdx] + mov r11,QWORD PTR[24+rcx] + mov QWORD PTR[32+rdi],r12 + sbb rbp,QWORD PTR[80+rdx] + mov r12,QWORD PTR[32+rcx] + mov QWORD PTR[40+rdi],r13 + sbb rsi,QWORD PTR[88+rdx] + mov r13,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r8,rdx + and r9,rdx + and r10,rdx + and r11,rdx + and r12,rdx + and r13,rdx + + add r14,r8 + adc r15,r9 + mov QWORD PTR[48+rdi],r14 + adc rax,r10 + mov QWORD PTR[56+rdi],r15 + adc rbx,r11 + mov QWORD PTR[64+rdi],rax + adc rbp,r12 + mov QWORD PTR[72+rdi],rbx + adc rsi,r13 + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rsi + + DB 0F3h,0C3h ;repret +__sub_mod_384x384 ENDP + +PUBLIC add_mod_384x384 + + +ALIGN 32 +add_mod_384x384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_add_mod_384x384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_add_mod_384x384:: + + + call __add_mod_384x384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_add_mod_384x384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_add_mod_384x384:: +add_mod_384x384 ENDP + +PUBLIC sub_mod_384x384 + + +ALIGN 32 +sub_mod_384x384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sub_mod_384x384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sub_mod_384x384:: + + + call __sub_mod_384x384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sub_mod_384x384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sub_mod_384x384:: +sub_mod_384x384 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_add_mod_384x384 + DD imagerel $L$SEH_body_add_mod_384x384 + DD imagerel $L$SEH_info_add_mod_384x384_prologue + + DD imagerel $L$SEH_body_add_mod_384x384 + DD imagerel $L$SEH_epilogue_add_mod_384x384 + DD imagerel $L$SEH_info_add_mod_384x384_body + + DD imagerel $L$SEH_epilogue_add_mod_384x384 + DD imagerel $L$SEH_end_add_mod_384x384 + DD imagerel $L$SEH_info_add_mod_384x384_epilogue + + DD imagerel $L$SEH_begin_sub_mod_384x384 + DD imagerel $L$SEH_body_sub_mod_384x384 + DD imagerel $L$SEH_info_sub_mod_384x384_prologue + + DD imagerel $L$SEH_body_sub_mod_384x384 + DD imagerel $L$SEH_epilogue_sub_mod_384x384 + DD imagerel $L$SEH_info_sub_mod_384x384_body + + DD imagerel $L$SEH_epilogue_sub_mod_384x384 + DD imagerel $L$SEH_end_sub_mod_384x384 + DD imagerel $L$SEH_info_sub_mod_384x384_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_add_mod_384x384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_add_mod_384x384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_add_mod_384x384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sub_mod_384x384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sub_mod_384x384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sub_mod_384x384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/blst.def b/crypto/blst_src/build/win64/blst.def new file mode 100644 index 00000000000..3fbb6b3a97d --- /dev/null +++ b/crypto/blst_src/build/win64/blst.def @@ -0,0 +1,217 @@ +LIBRARY blst + +EXPORTS + blst_scalar_from_uint32 + blst_uint32_from_scalar + blst_scalar_from_uint64 + blst_uint64_from_scalar + blst_scalar_from_bendian + blst_bendian_from_scalar + blst_scalar_from_lendian + blst_lendian_from_scalar + blst_scalar_fr_check + blst_sk_check + blst_sk_add_n_check + blst_sk_sub_n_check + blst_sk_mul_n_check + blst_sk_inverse + blst_scalar_from_le_bytes + blst_scalar_from_be_bytes + blst_fr_add + blst_fr_sub + blst_fr_mul_by_3 + blst_fr_lshift + blst_fr_rshift + blst_fr_mul + blst_fr_sqr + blst_fr_cneg + blst_fr_eucl_inverse + blst_fr_inverse + blst_fr_from_uint64 + blst_uint64_from_fr + blst_fr_from_scalar + blst_scalar_from_fr + blst_fp_add + blst_fp_sub + blst_fp_mul_by_3 + blst_fp_mul_by_8 + blst_fp_lshift + blst_fp_mul + blst_fp_sqr + blst_fp_cneg + blst_fp_eucl_inverse + blst_fp_inverse + blst_fp_sqrt + blst_fp_from_uint32 + blst_uint32_from_fp + blst_fp_from_uint64 + blst_uint64_from_fp + blst_fp_from_bendian + blst_bendian_from_fp + blst_fp_from_lendian + blst_lendian_from_fp + blst_fp2_add + blst_fp2_sub + blst_fp2_mul_by_3 + blst_fp2_mul_by_8 + blst_fp2_lshift + blst_fp2_mul + blst_fp2_sqr + blst_fp2_cneg + blst_fp2_eucl_inverse + blst_fp2_inverse + blst_fp2_sqrt + blst_fp12_sqr + blst_fp12_cyclotomic_sqr + blst_fp12_mul + blst_fp12_mul_by_xy00z0 + blst_fp12_conjugate + blst_fp12_inverse + blst_fp12_frobenius_map + blst_fp12_is_equal + blst_fp12_is_one + blst_fp12_in_group + blst_fp12_one + blst_p1_add + blst_p1_add_or_double + blst_p1_add_affine + blst_p1_add_or_double_affine + blst_p1_double + blst_p1_mult + blst_p1_cneg + blst_p1_to_affine + blst_p1_from_affine + blst_p1_on_curve + blst_p1_in_g1 + blst_p1_is_equal + blst_p1_is_inf + blst_p1_generator + blst_p1_affine_on_curve + blst_p1_affine_in_g1 + blst_p1_affine_is_equal + blst_p1_affine_is_inf + blst_p1_affine_generator + blst_p2_add + blst_p2_add_or_double + blst_p2_add_affine + blst_p2_add_or_double_affine + blst_p2_double + blst_p2_mult + blst_p2_cneg + blst_p2_to_affine + blst_p2_from_affine + blst_p2_on_curve + blst_p2_in_g2 + blst_p2_is_equal + blst_p2_is_inf + blst_p2_generator + blst_p2_affine_on_curve + blst_p2_affine_in_g2 + blst_p2_affine_is_equal + blst_p2_affine_is_inf + blst_p2_affine_generator + blst_p1s_to_affine + blst_p1s_add + blst_p1s_mult_wbits_precompute_sizeof + blst_p1s_mult_wbits_precompute + blst_p1s_mult_wbits_scratch_sizeof + blst_p1s_mult_wbits + blst_p1s_mult_pippenger_scratch_sizeof + blst_p1s_mult_pippenger + blst_p1s_tile_pippenger + blst_p2s_to_affine + blst_p2s_add + blst_p2s_mult_wbits_precompute_sizeof + blst_p2s_mult_wbits_precompute + blst_p2s_mult_wbits_scratch_sizeof + blst_p2s_mult_wbits + blst_p2s_mult_pippenger_scratch_sizeof + blst_p2s_mult_pippenger + blst_p2s_tile_pippenger + blst_map_to_g1 + blst_map_to_g2 + blst_encode_to_g1 + blst_hash_to_g1 + blst_encode_to_g2 + blst_hash_to_g2 + blst_p1_serialize + blst_p1_compress + blst_p1_affine_serialize + blst_p1_affine_compress + blst_p1_uncompress + blst_p1_deserialize + blst_p2_serialize + blst_p2_compress + blst_p2_affine_serialize + blst_p2_affine_compress + blst_p2_uncompress + blst_p2_deserialize + blst_keygen + blst_sk_to_pk_in_g1 + blst_sign_pk_in_g1 + blst_sk_to_pk_in_g2 + blst_sign_pk_in_g2 + blst_miller_loop + blst_final_exp + blst_precompute_lines + blst_miller_loop_lines + blst_fp12_finalverify + blst_pairing_sizeof + blst_pairing_init + blst_pairing_get_dst + blst_pairing_commit + blst_pairing_aggregate_pk_in_g2 + blst_pairing_chk_n_aggr_pk_in_g2 + blst_pairing_mul_n_aggregate_pk_in_g2 + blst_pairing_chk_n_mul_n_aggr_pk_in_g2 + blst_pairing_aggregate_pk_in_g1 + blst_pairing_chk_n_aggr_pk_in_g1 + blst_pairing_mul_n_aggregate_pk_in_g1 + blst_pairing_chk_n_mul_n_aggr_pk_in_g1 + blst_pairing_merge + blst_pairing_finalverify + blst_aggregate_in_g1 + blst_aggregate_in_g2 + blst_aggregated_in_g1 + blst_aggregated_in_g2 + blst_core_verify_pk_in_g1 + blst_core_verify_pk_in_g2 + BLS12_381_G1 + BLS12_381_NEG_G1 + BLS12_381_G2 + BLS12_381_NEG_G2 + blst_fr_to + blst_fr_from + blst_fp_to + blst_fp_from + blst_fp_is_square + blst_fp2_is_square + blst_p1_from_jacobian + blst_p2_from_jacobian + blst_sk_to_pk2_in_g1 + blst_sign_pk2_in_g1 + blst_sk_to_pk2_in_g2 + blst_sign_pk2_in_g2 + blst_uniq_sizeof + blst_uniq_init + blst_uniq_test + blst_expand_message_xmd + blst_p1_unchecked_mult + blst_p2_unchecked_mult + blst_pairing_raw_aggregate + blst_pairing_as_fp12 + blst_bendian_from_fp12 + blst_keygen_v3 + blst_keygen_v4_5 + blst_keygen_v5 + blst_derive_master_eip2333 + blst_derive_child_eip2333 + blst_scalar_from_hexascii + blst_fr_from_hexascii + blst_fp_from_hexascii + blst_p1_sizeof + blst_p1_affine_sizeof + blst_p2_sizeof + blst_p2_affine_sizeof + blst_fp12_sizeof + diff --git a/crypto/blst_src/build/win64/ct_inverse_mod_256-armv8.asm b/crypto/blst_src/build/win64/ct_inverse_mod_256-armv8.asm new file mode 100644 index 00000000000..f3c2f0d05f9 --- /dev/null +++ b/crypto/blst_src/build/win64/ct_inverse_mod_256-armv8.asm @@ -0,0 +1,785 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + EXPORT |ct_inverse_mod_256|[FUNC] + ALIGN 32 +|ct_inverse_mod_256| PROC + DCDU 3573752639 + stp x29, x30, [sp,#-80]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + sub sp, sp, #1040 + + ldp x4, x5, [x1,#8*0] + ldp x6, x7, [x1,#8*2] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + str x0, [sp] + + ldp x8, x9, [x2,#8*0] + ldp x10, x11, [x2,#8*2] + + stp x4, x5, [x1,#8*0] // copy input to |a| + stp x6, x7, [x1,#8*2] + stp x8, x9, [x1,#8*4] // copy modulus to |b| + stp x10, x11, [x1,#8*6] + + ////////////////////////////////////////// first iteration + bl |$Lab_approximation_31_256_loaded| + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + str x12,[x0,#8*8] // initialize |u| with |f0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to dst |b| + bl __smul_256_n_shift_by_31 + str x12, [x0,#8*9] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + ldr x8, [x1,#8*8] // |u| + ldr x9, [x1,#8*13] // |v| + madd x4, x16, x8, xzr // |u|*|f0| + madd x4, x17, x9, x4 // |v|*|g0| + str x4, [x0,#8*4] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*5] + stp x5, x5, [x0,#8*7] + + madd x4, x12, x8, xzr // |u|*|f1| + madd x4, x13, x9, x4 // |v|*|g1| + str x4, [x0,#8*9] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*10] + stp x5, x5, [x0,#8*12] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + adc x22, x22, x23 + stp x22, x22, [x0,#8*4] + stp x22, x22, [x0,#8*6] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + bl __ab_approximation_31_256 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_256_n_shift_by_31 + mov x16, x12 // corrected |f0| + mov x17, x13 // corrected |g0| + + mov x12, x14 // |f1| + mov x13, x15 // |g1| + add x0, x0, #8*4 // pointer to destination |b| + bl __smul_256_n_shift_by_31 + + add x0, x0, #8*4 // pointer to destination |u| + bl __smul_256x63 + adc x22, x22, x23 + str x22, [x0,#8*4] + + mov x16, x12 // corrected |f1| + mov x17, x13 // corrected |g1| + add x0, x0, #8*5 // pointer to destination |v| + bl __smul_256x63 + bl __smul_512x63_tail + ////////////////////////////////////////// two[!] last iterations + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #47 // 31 + 512 % 31 + //bl __ab_approximation_62_256 // |a| and |b| are exact, + ldr x7, [x1,#8*0] // just load + ldr x11, [x1,#8*4] + bl __inner_loop_62_256 + + mov x16, x14 + mov x17, x15 + ldr x0, [sp] // original out_ptr + bl __smul_256x63 + bl __smul_512x63_tail + ldr x30, [x29,#8] + + smulh x20, x7, x17 // figure out top-most limb + ldp x8, x9, [x3,#8*0] + adc x23, x23, x25 + ldp x10, x11, [x3,#8*2] + + add x20, x20, x23 // x20 is 1, 0 or -1 + asr x19, x20, #63 // sign as mask + + and x23, x8, x19 // add mod<<256 conditionally + and x24, x9, x19 + adds x4, x4, x23 + and x25, x10, x19 + adcs x5, x5, x24 + and x26, x11, x19 + adcs x6, x6, x25 + adcs x7, x22, x26 + adc x20, x20, xzr // x20 is 1, 0 or -1 + + neg x19, x20 + orr x20, x20, x19 // excess bit or sign as mask + asr x19, x19, #63 // excess bit as mask + + and x8, x8, x20 // mask |mod| + and x9, x9, x20 + and x10, x10, x20 + and x11, x11, x20 + + eor x8, x8, x19 // conditionally negate |mod| + eor x9, x9, x19 + adds x8, x8, x19, lsr#63 + eor x10, x10, x19 + adcs x9, x9, xzr + eor x11, x11, x19 + adcs x10, x10, xzr + adc x11, x11, xzr + + adds x4, x4, x8 // final adjustment for |mod|<<256 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*4] + adc x7, x7, x11 + stp x6, x7, [x0,#8*6] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldr x29, [sp],#80 + DCDU 3573752767 + ret + ENDP + +//////////////////////////////////////////////////////////////////////// + + ALIGN 32 +|__smul_256x63| PROC + ldp x4, x5, [x1,#8*0+64] // load |u| (or |v|) + asr x14, x16, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x6, x7, [x1,#8*2+64] + eor x16, x16, x14 // conditionally negate |f_| (or |g_|) + ldr x22, [x1,#8*4+64] + + eor x4, x4, x14 // conditionally negate |u| (or |v|) + sub x16, x16, x14 + eor x5, x5, x14 + adds x4, x4, x14, lsr#63 + eor x6, x6, x14 + adcs x5, x5, xzr + eor x7, x7, x14 + adcs x6, x6, xzr + eor x22, x22, x14 + umulh x19, x4, x16 + adcs x7, x7, xzr + umulh x20, x5, x16 + adcs x22, x22, xzr + umulh x21, x6, x16 + mul x4, x4, x16 + cmp x16, #0 + mul x5, x5, x16 + cselne x22,x22,xzr + mul x6, x6, x16 + adds x5, x5, x19 + mul x24, x7, x16 + adcs x6, x6, x20 + adcs x24, x24, x21 + adc x26, xzr, xzr + ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x10, x11, [x1,#8*2+104] + eor x17, x17, x14 // conditionally negate |f_| (or |g_|) + ldr x23, [x1,#8*4+104] + + eor x8, x8, x14 // conditionally negate |u| (or |v|) + sub x17, x17, x14 + eor x9, x9, x14 + adds x8, x8, x14, lsr#63 + eor x10, x10, x14 + adcs x9, x9, xzr + eor x11, x11, x14 + adcs x10, x10, xzr + eor x23, x23, x14 + umulh x19, x8, x17 + adcs x11, x11, xzr + umulh x20, x9, x17 + adcs x23, x23, xzr + umulh x21, x10, x17 + adc x15, xzr, xzr // used in __smul_512x63_tail + mul x8, x8, x17 + cmp x17, #0 + mul x9, x9, x17 + cselne x23,x23,xzr + mul x10, x10, x17 + adds x9, x9, x19 + mul x25, x11, x17 + adcs x10, x10, x20 + adcs x25, x25, x21 + adc x26, x26, xzr + + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + stp x4, x5, [x0,#8*0] + adcs x24, x24, x25 + stp x6, x24, [x0,#8*2] + + ret + ENDP + + + ALIGN 32 +|__smul_512x63_tail| PROC + umulh x24, x7, x16 + ldp x5, x6, [x1,#8*18] // load rest of |v| + adc x26, x26, xzr + ldr x7, [x1,#8*20] + and x22, x22, x16 + + umulh x11, x11, x17 // resume |v|*|g1| chain + + sub x24, x24, x22 // tie up |u|*|f1| chain + asr x25, x24, #63 + + eor x5, x5, x14 // conditionally negate rest of |v| + eor x6, x6, x14 + adds x5, x5, x15 + eor x7, x7, x14 + adcs x6, x6, xzr + umulh x19, x23, x17 + adc x7, x7, xzr + umulh x20, x5, x17 + add x11, x11, x26 + umulh x21, x6, x17 + + mul x4, x23, x17 + mul x5, x5, x17 + adds x4, x4, x11 + mul x6, x6, x17 + adcs x5, x5, x19 + mul x22, x7, x17 + adcs x6, x6, x20 + adcs x22, x22, x21 + adc x23, xzr, xzr // used in the final step + + adds x4, x4, x24 + adcs x5, x5, x25 + adcs x6, x6, x25 + stp x4, x5, [x0,#8*4] + adcs x22, x22, x25 // carry is used in the final step + stp x6, x22, [x0,#8*6] + + ret + ENDP + + + ALIGN 32 +|__smul_256_n_shift_by_31| PROC + ldp x4, x5, [x1,#8*0+0] // load |a| (or |b|) + asr x24, x12, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x6, x7, [x1,#8*2+0] + eor x25, x12, x24 // conditionally negate |f0| (or |g0|) + + eor x4, x4, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x5, x5, x24 + adds x4, x4, x24, lsr#63 + eor x6, x6, x24 + adcs x5, x5, xzr + eor x7, x7, x24 + umulh x19, x4, x25 + adcs x6, x6, xzr + umulh x20, x5, x25 + adc x7, x7, xzr + umulh x21, x6, x25 + and x24, x24, x25 + umulh x22, x7, x25 + neg x24, x24 + + mul x4, x4, x25 + mul x5, x5, x25 + mul x6, x6, x25 + adds x5, x5, x19 + mul x7, x7, x25 + adcs x6, x6, x20 + adcs x7, x7, x21 + adc x22, x22, x24 + ldp x8, x9, [x1,#8*0+32] // load |a| (or |b|) + asr x24, x13, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x10, x11, [x1,#8*2+32] + eor x25, x13, x24 // conditionally negate |f0| (or |g0|) + + eor x8, x8, x24 // conditionally negate |a| (or |b|) + sub x25, x25, x24 + eor x9, x9, x24 + adds x8, x8, x24, lsr#63 + eor x10, x10, x24 + adcs x9, x9, xzr + eor x11, x11, x24 + umulh x19, x8, x25 + adcs x10, x10, xzr + umulh x20, x9, x25 + adc x11, x11, xzr + umulh x21, x10, x25 + and x24, x24, x25 + umulh x23, x11, x25 + neg x24, x24 + + mul x8, x8, x25 + mul x9, x9, x25 + mul x10, x10, x25 + adds x9, x9, x19 + mul x11, x11, x25 + adcs x10, x10, x20 + adcs x11, x11, x21 + adc x23, x23, x24 + adds x4, x4, x8 + adcs x5, x5, x9 + adcs x6, x6, x10 + adcs x7, x7, x11 + adc x8, x22, x23 + + extr x4, x5, x4, #31 + extr x5, x6, x5, #31 + extr x6, x7, x6, #31 + asr x23, x8, #63 // result's sign as mask + extr x7, x8, x7, #31 + + eor x4, x4, x23 // ensure the result is positive + eor x5, x5, x23 + adds x4, x4, x23, lsr#63 + eor x6, x6, x23 + adcs x5, x5, xzr + eor x7, x7, x23 + adcs x6, x6, xzr + stp x4, x5, [x0,#8*0] + adc x7, x7, xzr + stp x6, x7, [x0,#8*2] + + eor x12, x12, x23 // adjust |f/g| accordingly + eor x13, x13, x23 + sub x12, x12, x23 + sub x13, x13, x23 + + ret + ENDP + + ALIGN 16 +|__ab_approximation_31_256| PROC + ldp x6, x7, [x1,#8*2] + ldp x10, x11, [x1,#8*6] + ldp x4, x5, [x1,#8*0] + ldp x8, x9, [x1,#8*4] + +|$Lab_approximation_31_256_loaded| + orr x19, x7, x11 // check top-most limbs, ... + cmp x19, #0 + cselne x7,x7,x6 + cselne x11,x11,x10 + cselne x6,x6,x5 + orr x19, x7, x11 // and ones before top-most, ... + cselne x10,x10,x9 + + cmp x19, #0 + cselne x7,x7,x6 + cselne x11,x11,x10 + cselne x6,x6,x4 + orr x19, x7, x11 // and one more, ... + cselne x10,x10,x8 + + clz x19, x19 + cmp x19, #64 + cselne x19,x19,xzr + cselne x7,x7,x6 + cselne x11,x11,x10 + neg x20, x19 + + lslv x7, x7, x19 // align high limbs to the left + lslv x11, x11, x19 + lsrv x6, x6, x20 + lsrv x10, x10, x20 + and x6, x6, x20, asr#6 + and x10, x10, x20, asr#6 + orr x7, x7, x6 + orr x11, x11, x10 + + bfxil x7, x4, #0, #31 + bfxil x11, x8, #0, #31 + + b __inner_loop_31_256 + ret + ENDP + + + ALIGN 16 +|__inner_loop_31_256| PROC + mov x2, #31 + mov x13, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x15, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x23,#0x7FFFFFFF7FFFFFFF + +|$Loop_31_256| + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x15 + cselhs x11,x11,x7 + cselhs x7,x21,x20 + cselhs x15,x15,x13 + cselhs x13,x13,x19 + lsr x7, x7, #1 + and x19, x15, x22 + and x20, x23, x22 + sub x13, x13, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x15, x15, x15 // |f1|<<=1 + add x13, x13, x20 + sub x15, x15, x23 + cbnz x2, |$Loop_31_256| + + mov x23, #0x7FFFFFFF + ubfx x12, x13, #0, #32 + ubfx x13, x13, #32, #32 + ubfx x14, x15, #0, #32 + ubfx x15, x15, #32, #32 + sub x12, x12, x23 // remove bias + sub x13, x13, x23 + sub x14, x14, x23 + sub x15, x15, x23 + + ret + ENDP + + + ALIGN 16 +|__inner_loop_62_256| PROC + mov x12, #1 // |f0|=1 + mov x13, #0 // |g0|=0 + mov x14, #0 // |f1|=0 + mov x15, #1 // |g1|=1 + +|$Loop_62_256| + sbfx x22, x7, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + and x19, x11, x22 + sub x20, x11, x7 // |b_|-|a_| + subs x21, x7, x19 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x19, x12 + cselhs x11,x11,x7 + cselhs x7,x21,x20 + mov x20, x13 + cselhs x12,x12,x14 + cselhs x14,x14,x19 + cselhs x13,x13,x15 + cselhs x15,x15,x20 + lsr x7, x7, #1 + and x19, x14, x22 + and x20, x15, x22 + add x14, x14, x14 // |f1|<<=1 + add x15, x15, x15 // |g1|<<=1 + sub x12, x12, x19 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x13, x13, x20 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, |$Loop_62_256| + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/ct_inverse_mod_256-x86_64.asm b/crypto/blst_src/build/win64/ct_inverse_mod_256-x86_64.asm new file mode 100644 index 00000000000..65665c9f17a --- /dev/null +++ b/crypto/blst_src/build/win64/ct_inverse_mod_256-x86_64.asm @@ -0,0 +1,1211 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC ct_inverse_mod_256 + +ALIGN 32 +ct_inverse_mod_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_ct_inverse_mod_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,1072 + +$L$SEH_body_ct_inverse_mod_256:: + + + lea rax,QWORD PTR[((48+511))+rsp] + and rax,-512 + mov QWORD PTR[32+rsp],rdi + mov QWORD PTR[40+rsp],rcx + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + + mov r12,QWORD PTR[rdx] + mov r13,QWORD PTR[8+rdx] + mov r14,QWORD PTR[16+rdx] + mov r15,QWORD PTR[24+rdx] + + mov QWORD PTR[rax],r8 + mov QWORD PTR[8+rax],r9 + mov QWORD PTR[16+rax],r10 + mov QWORD PTR[24+rax],r11 + + mov QWORD PTR[32+rax],r12 + mov QWORD PTR[40+rax],r13 + mov QWORD PTR[48+rax],r14 + mov QWORD PTR[56+rax],r15 + mov rsi,rax + + + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + + + mov QWORD PTR[64+rdi],rdx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + + + mov QWORD PTR[72+rdi],rdx + + + xor rsi,256 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + + + + mov r8,QWORD PTR[64+rsi] + mov r12,QWORD PTR[104+rsi] + mov r9,r8 + imul r8,QWORD PTR[rsp] + mov r13,r12 + imul r12,QWORD PTR[8+rsp] + add r8,r12 + mov QWORD PTR[32+rdi],r8 + sar r8,63 + mov QWORD PTR[40+rdi],r8 + mov QWORD PTR[48+rdi],r8 + mov QWORD PTR[56+rdi],r8 + mov QWORD PTR[64+rdi],r8 + lea rsi,QWORD PTR[64+rsi] + + imul r9,rdx + imul r13,rcx + add r9,r13 + mov QWORD PTR[72+rdi],r9 + sar r9,63 + mov QWORD PTR[80+rdi],r9 + mov QWORD PTR[88+rdi],r9 + mov QWORD PTR[96+rdi],r9 + mov QWORD PTR[104+rdi],r9 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_256x63 + sar rbp,63 + mov QWORD PTR[40+rdi],rbp + mov QWORD PTR[48+rdi],rbp + mov QWORD PTR[56+rdi],rbp + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + xor rsi,256+8*8 + mov edx,31 + call __ab_approximation_31_256 + + + mov QWORD PTR[16+rsp],r12 + mov QWORD PTR[24+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_256_n_shift_by_31 + mov QWORD PTR[rsp],rdx + mov QWORD PTR[8+rsp],rcx + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256_n_shift_by_31 + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[24+rsp],rcx + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[64+rsi] + lea rdi,QWORD PTR[32+rdi] + call __smulq_256x63 + + mov rdx,QWORD PTR[16+rsp] + mov rcx,QWORD PTR[24+rsp] + lea rdi,QWORD PTR[40+rdi] + call __smulq_512x63 + + xor rsi,256+8*8 + mov edx,47 + + mov r8,QWORD PTR[rsi] + + mov r10,QWORD PTR[32+rsi] + + call __inner_loop_62_256 + + + + + + + + lea rsi,QWORD PTR[64+rsi] + + + + + + mov rdx,r12 + mov rcx,r13 + mov rdi,QWORD PTR[32+rsp] + call __smulq_512x63 + adc rdx,rbp + + mov rsi,QWORD PTR[40+rsp] + mov rax,rdx + sar rdx,63 + + mov r8,rdx + mov r9,rdx + and r8,QWORD PTR[rsi] + mov r10,rdx + and r9,QWORD PTR[8+rsi] + and r10,QWORD PTR[16+rsi] + and rdx,QWORD PTR[24+rsi] + + add r12,r8 + adc r13,r9 + adc r14,r10 + adc r15,rdx + adc rax,0 + + mov rdx,rax + neg rax + or rdx,rax + sar rax,63 + + mov r8,rdx + mov r9,rdx + and r8,QWORD PTR[rsi] + mov r10,rdx + and r9,QWORD PTR[8+rsi] + and r10,QWORD PTR[16+rsi] + and rdx,QWORD PTR[24+rsi] + + xor r8,rax + xor rcx,rcx + xor r9,rax + sub rcx,rax + xor r10,rax + xor rdx,rax + add r8,rcx + adc r9,0 + adc r10,0 + adc rdx,0 + + add r12,r8 + adc r13,r9 + adc r14,r10 + adc r15,rdx + + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + + lea r8,QWORD PTR[1072+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_ct_inverse_mod_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_ct_inverse_mod_256:: +ct_inverse_mod_256 ENDP + +ALIGN 32 +__smulq_512x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov rbp,QWORD PTR[32+rsi] + + mov rbx,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbx,rdx + add rbx,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor rbp,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc rbp,0 + + mul rbx + mov QWORD PTR[rdi],rax + mov rax,r9 + mov r9,rdx + mul rbx + add r9,rax + mov rax,r10 + adc rdx,0 + mov QWORD PTR[8+rdi],r9 + mov r10,rdx + mul rbx + add r10,rax + mov rax,r11 + adc rdx,0 + mov QWORD PTR[16+rdi],r10 + mov r11,rdx + and rbp,rbx + neg rbp + mul rbx + add r11,rax + adc rbp,rdx + mov QWORD PTR[24+rdi],r11 + + mov r8,QWORD PTR[40+rsi] + mov r9,QWORD PTR[48+rsi] + mov r10,QWORD PTR[56+rsi] + mov r11,QWORD PTR[64+rsi] + mov r12,QWORD PTR[72+rsi] + mov r13,QWORD PTR[80+rsi] + mov r14,QWORD PTR[88+rsi] + mov r15,QWORD PTR[96+rsi] + + mov rdx,rcx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rcx,rdx + add rcx,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + xor r14,rdx + xor r15,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + adc r15,0 + + mul rcx + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rcx + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rcx + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rcx + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rcx + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + mul rcx + add r13,rax + mov rax,r14 + adc rdx,0 + mov r14,rdx + mul rcx + add r14,rax + mov rax,r15 + adc rdx,0 + mov r15,rdx + imul rcx + add r15,rax + adc rdx,0 + + mov rbx,rbp + sar rbp,63 + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,rbx + adc r13,rbp + adc r14,rbp + adc r15,rbp + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + + DB 0F3h,0C3h ;repret +__smulq_512x63 ENDP + + +ALIGN 32 +__smulq_256x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[((0+0))+rsi] + mov r9,QWORD PTR[((0+8))+rsi] + mov r10,QWORD PTR[((0+16))+rsi] + mov r11,QWORD PTR[((0+24))+rsi] + mov rbp,QWORD PTR[((0+32))+rsi] + + mov rbx,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbx,rdx + add rbx,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor rbp,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc rbp,0 + + mul rbx + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbx + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbx + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + and rbp,rbx + neg rbp + mul rbx + add r11,rax + adc rbp,rdx + mov rdx,rcx + mov r12,QWORD PTR[((40+0))+rsi] + mov r13,QWORD PTR[((40+8))+rsi] + mov r14,QWORD PTR[((40+16))+rsi] + mov r15,QWORD PTR[((40+24))+rsi] + mov rcx,QWORD PTR[((40+32))+rsi] + + mov rbx,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbx,rdx + add rbx,rax + + xor r12,rdx + xor r13,rdx + xor r14,rdx + xor r15,rdx + xor rcx,rdx + add rax,r12 + adc r13,0 + adc r14,0 + adc r15,0 + adc rcx,0 + + mul rbx + mov r12,rax + mov rax,r13 + mov r13,rdx + mul rbx + add r13,rax + mov rax,r14 + adc rdx,0 + mov r14,rdx + mul rbx + add r14,rax + mov rax,r15 + adc rdx,0 + mov r15,rdx + and rcx,rbx + neg rcx + mul rbx + add r15,rax + adc rcx,rdx + add r8,r12 + adc r9,r13 + adc r10,r14 + adc r11,r15 + adc rbp,rcx + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],rbp + + DB 0F3h,0C3h ;repret +__smulq_256x63 ENDP + +ALIGN 32 +__smulq_256_n_shift_by_31 PROC PRIVATE + DB 243,15,30,250 + mov QWORD PTR[rdi],rdx + mov QWORD PTR[8+rdi],rcx + mov rbp,rdx + mov r8,QWORD PTR[((0+0))+rsi] + mov r9,QWORD PTR[((0+8))+rsi] + mov r10,QWORD PTR[((0+16))+rsi] + mov r11,QWORD PTR[((0+24))+rsi] + + mov rbx,rbp + sar rbp,63 + xor rax,rax + sub rax,rbp + + xor rbx,rbp + add rbx,rax + + xor r8,rbp + xor r9,rbp + xor r10,rbp + xor r11,rbp + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + + mul rbx + mov r8,rax + mov rax,r9 + and rbp,rbx + neg rbp + mov r9,rdx + mul rbx + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbx + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbx + add r11,rax + adc rbp,rdx + mov r12,QWORD PTR[((32+0))+rsi] + mov r13,QWORD PTR[((32+8))+rsi] + mov r14,QWORD PTR[((32+16))+rsi] + mov r15,QWORD PTR[((32+24))+rsi] + + mov rbx,rcx + sar rcx,63 + xor rax,rax + sub rax,rcx + + xor rbx,rcx + add rbx,rax + + xor r12,rcx + xor r13,rcx + xor r14,rcx + xor r15,rcx + add rax,r12 + adc r13,0 + adc r14,0 + adc r15,0 + + mul rbx + mov r12,rax + mov rax,r13 + and rcx,rbx + neg rcx + mov r13,rdx + mul rbx + add r13,rax + mov rax,r14 + adc rdx,0 + mov r14,rdx + mul rbx + add r14,rax + mov rax,r15 + adc rdx,0 + mov r15,rdx + mul rbx + add r15,rax + adc rcx,rdx + add r8,r12 + adc r9,r13 + adc r10,r14 + adc r11,r15 + adc rbp,rcx + + mov rdx,QWORD PTR[rdi] + mov rcx,QWORD PTR[8+rdi] + + shrd r8,r9,31 + shrd r9,r10,31 + shrd r10,r11,31 + shrd r11,rbp,31 + + sar rbp,63 + xor rax,rax + sub rax,rbp + + xor r8,rbp + xor r9,rbp + xor r10,rbp + xor r11,rbp + add r8,rax + adc r9,0 + adc r10,0 + adc r11,0 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + xor rdx,rbp + xor rcx,rbp + add rdx,rax + add rcx,rax + + DB 0F3h,0C3h ;repret +__smulq_256_n_shift_by_31 ENDP + +ALIGN 32 +__ab_approximation_31_256 PROC PRIVATE + DB 243,15,30,250 + mov r9,QWORD PTR[24+rsi] + mov r11,QWORD PTR[56+rsi] + mov rbx,QWORD PTR[16+rsi] + mov rbp,QWORD PTR[48+rsi] + mov r8,QWORD PTR[8+rsi] + mov r10,QWORD PTR[40+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + mov r8,QWORD PTR[rsi] + cmovz rbp,r10 + mov r10,QWORD PTR[32+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + cmovz rbp,r10 + + mov rax,r9 + or rax,r11 + bsr rcx,rax + lea rcx,QWORD PTR[1+rcx] + cmovz r9,r8 + cmovz r11,r10 + cmovz rcx,rax + neg rcx + + + shld r9,rbx,cl + shld r11,rbp,cl + + mov eax,07FFFFFFFh + and r8,rax + and r10,rax + not rax + and r9,rax + and r11,rax + or r8,r9 + or r10,r11 + + jmp __inner_loop_31_256 + + DB 0F3h,0C3h ;repret +__ab_approximation_31_256 ENDP + +ALIGN 32 +__inner_loop_31_256 PROC PRIVATE + DB 243,15,30,250 + mov rcx,07FFFFFFF80000000h + mov r13,0800000007FFFFFFFh + mov r15,07FFFFFFF7FFFFFFFh + +$L$oop_31_256:: + cmp r8,r10 + mov rax,r8 + mov rbx,r10 + mov rbp,rcx + mov r14,r13 + cmovb r8,r10 + cmovb r10,rax + cmovb rcx,r13 + cmovb r13,rbp + + sub r8,r10 + sub rcx,r13 + add rcx,r15 + + test rax,1 + cmovz r8,rax + cmovz r10,rbx + cmovz rcx,rbp + cmovz r13,r14 + + shr r8,1 + add r13,r13 + sub r13,r15 + sub edx,1 + jnz $L$oop_31_256 + + shr r15,32 + mov edx,ecx + mov r12d,r13d + shr rcx,32 + shr r13,32 + sub rdx,r15 + sub rcx,r15 + sub r12,r15 + sub r13,r15 + + DB 0F3h,0C3h ;repret +__inner_loop_31_256 ENDP + + +ALIGN 32 +__inner_loop_62_256 PROC PRIVATE + DB 243,15,30,250 + mov r15d,edx + mov rdx,1 + xor rcx,rcx + xor r12,r12 + mov r13,rdx + mov r14,rdx + +$L$oop_62_256:: + xor rax,rax + test r8,r14 + mov rbx,r10 + cmovnz rax,r10 + sub rbx,r8 + mov rbp,r8 + sub r8,rax + cmovc r8,rbx + cmovc r10,rbp + mov rax,rdx + cmovc rdx,r12 + cmovc r12,rax + mov rbx,rcx + cmovc rcx,r13 + cmovc r13,rbx + xor rax,rax + xor rbx,rbx + shr r8,1 + test rbp,r14 + cmovnz rax,r12 + cmovnz rbx,r13 + add r12,r12 + add r13,r13 + sub rdx,rax + sub rcx,rbx + sub r15d,1 + jnz $L$oop_62_256 + + DB 0F3h,0C3h ;repret +__inner_loop_62_256 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_ct_inverse_mod_256 + DD imagerel $L$SEH_body_ct_inverse_mod_256 + DD imagerel $L$SEH_info_ct_inverse_mod_256_prologue + + DD imagerel $L$SEH_body_ct_inverse_mod_256 + DD imagerel $L$SEH_epilogue_ct_inverse_mod_256 + DD imagerel $L$SEH_info_ct_inverse_mod_256_body + + DD imagerel $L$SEH_epilogue_ct_inverse_mod_256 + DD imagerel $L$SEH_end_ct_inverse_mod_256 + DD imagerel $L$SEH_info_ct_inverse_mod_256_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_ct_inverse_mod_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_ct_inverse_mod_256_body:: +DB 1,0,18,0 +DB 000h,0f4h,086h,000h +DB 000h,0e4h,087h,000h +DB 000h,0d4h,088h,000h +DB 000h,0c4h,089h,000h +DB 000h,034h,08ah,000h +DB 000h,054h,08bh,000h +DB 000h,074h,08dh,000h +DB 000h,064h,08eh,000h +DB 000h,001h,08ch,000h +$L$SEH_info_ct_inverse_mod_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/ct_inverse_mod_384-armv8.asm b/crypto/blst_src/build/win64/ct_inverse_mod_384-armv8.asm new file mode 100644 index 00000000000..4ab12e052df --- /dev/null +++ b/crypto/blst_src/build/win64/ct_inverse_mod_384-armv8.asm @@ -0,0 +1,718 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + EXPORT |ct_inverse_mod_383|[FUNC] + ALIGN 32 +|ct_inverse_mod_383| PROC + DCDU 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #1040 + + ldp x22, x4, [x1,#8*0] + ldp x5, x6, [x1,#8*2] + ldp x7, x8, [x1,#8*4] + + add x1, sp, #16+511 // find closest 512-byte-aligned spot + and x1, x1, #-512 // in the frame... + stp x0, x3, [sp] + + ldp x9, x10, [x2,#8*0] + ldp x11, x12, [x2,#8*2] + ldp x13, x14, [x2,#8*4] + + stp x22, x4, [x1,#8*0] // copy input to |a| + stp x5, x6, [x1,#8*2] + stp x7, x8, [x1,#8*4] + stp x9, x10, [x1,#8*6] // copy modulus to |b| + stp x11, x12, [x1,#8*8] + stp x13, x14, [x1,#8*10] + + ////////////////////////////////////////// first iteration + mov x2, #62 + bl |$Lab_approximation_62_loaded| + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + str x15,[x0,#8*12] // initialize |u| with |f0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to dst |b| + bl __smul_383_n_shift_by_62 + str x15, [x0,#8*12] // initialize |v| with |f1| + + ////////////////////////////////////////// second iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + ldr x7, [x1,#8*12] // |u| + ldr x8, [x1,#8*18] // |v| + mul x3, x20, x7 // |u|*|f0| + smulh x4, x20, x7 + mul x5, x21, x8 // |v|*|g0| + smulh x6, x21, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*6] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*8] + stp x5, x5, [x0,#8*10] + + mul x3, x15, x7 // |u|*|f1| + smulh x4, x15, x7 + mul x5, x16, x8 // |v|*|g1| + smulh x6, x16, x8 + adds x3, x3, x5 + adc x4, x4, x6 + stp x3, x4, [x0,#8*12] + asr x5, x4, #63 // sign extenstion + stp x5, x5, [x0,#8*14] + stp x5, x5, [x0,#8*16] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + asr x27, x27, #63 // sign extension + stp x27, x27, [x0,#8*6] + stp x27, x27, [x0,#8*8] + stp x27, x27, [x0,#8*10] + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + bl __ab_approximation_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + bl __smul_383_n_shift_by_62 + mov x20, x15 // corrected |f0| + mov x21, x16 // corrected |g0| + + mov x15, x17 // |f1| + mov x16, x19 // |g1| + add x0, x0, #8*6 // pointer to destination |b| + bl __smul_383_n_shift_by_62 + + add x0, x0, #8*6 // pointer to destination |u| + bl __smul_383x63 + + mov x20, x15 // corrected |f1| + mov x21, x16 // corrected |g1| + add x0, x0, #8*6 // pointer to destination |v| + bl __smul_383x63 + bl __smul_767x63_tail + ////////////////////////////////////////// iteration before last + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldp x3, x8, [x1,#8*0] // just load + ldp x9, x14, [x1,#8*6] + bl __inner_loop_62 + + eor x0, x1, #256 // pointer to dst |a|b|u|v| + str x3, [x0,#8*0] + str x9, [x0,#8*6] + + mov x20, x15 // exact |f0| + mov x21, x16 // exact |g0| + mov x15, x17 + mov x16, x19 + add x0, x0, #8*12 // pointer to dst |u| + bl __smul_383x63 + + mov x20, x15 // exact |f1| + mov x21, x16 // exact |g1| + add x0, x0, #8*6 // pointer to dst |v| + bl __smul_383x63 + bl __smul_767x63_tail + + ////////////////////////////////////////// last iteration + eor x1, x1, #256 // flip-flop src |a|b|u|v| + mov x2, #22 // 766 % 62 + //bl __ab_approximation_62 // |a| and |b| are exact, + ldr x3, [x1,#8*0] // just load + eor x8, x8, x8 + ldr x9, [x1,#8*6] + eor x14, x14, x14 + bl __inner_loop_62 + + mov x20, x17 + mov x21, x19 + ldp x0, x15, [sp] // original out_ptr and n_ptr + bl __smul_383x63 + bl __smul_767x63_tail + ldr x30, [x29,#8] + + asr x22, x8, #63 // sign as mask + ldp x9, x10, [x15,#8*0] + ldp x11, x12, [x15,#8*2] + ldp x13, x14, [x15,#8*4] + + and x9, x9, x22 // add mod<<384 conditionally + and x10, x10, x22 + adds x3, x3, x9 + and x11, x11, x22 + adcs x4, x4, x10 + and x12, x12, x22 + adcs x5, x5, x11 + and x13, x13, x22 + adcs x6, x6, x12 + and x14, x14, x22 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*8] + adc x8, x8, x14 + stp x7, x8, [x0,#8*10] + + add sp, sp, #1040 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 + DCDU 3573752767 + ret + ENDP + +//////////////////////////////////////////////////////////////////////// +// see corresponding commentary in ctx_inverse_mod_384-x86_64... + + ALIGN 32 +|__smul_383x63| PROC + ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) + asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x5, x6, [x1,#8*2+96] + eor x20, x20, x17 // conditionally negate |f_| (or |g_|) + ldp x7, x8, [x1,#8*4+96] + + eor x3, x3, x17 // conditionally negate |u| (or |v|) + sub x20, x20, x17 + eor x4, x4, x17 + adds x3, x3, x17, lsr#63 + eor x5, x5, x17 + adcs x4, x4, xzr + eor x6, x6, x17 + adcs x5, x5, xzr + eor x7, x7, x17 + adcs x6, x6, xzr + umulh x22, x3, x20 + eor x8, x8, x17 + umulh x23, x4, x20 + adcs x7, x7, xzr + umulh x24, x5, x20 + adcs x8, x8, xzr + umulh x25, x6, x20 + umulh x26, x7, x20 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x22 + mul x6, x6, x20 + adcs x5, x5, x23 + mul x7, x7, x20 + adcs x6, x6, x24 + mul x27,x8, x20 + adcs x7, x7, x25 + adcs x27,x27,x26 + adc x2, xzr, xzr + ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) + ldp x11, x12, [x1,#8*2+144] + eor x21, x21, x17 // conditionally negate |f_| (or |g_|) + ldp x13, x14, [x1,#8*4+144] + + eor x9, x9, x17 // conditionally negate |u| (or |v|) + sub x21, x21, x17 + eor x10, x10, x17 + adds x9, x9, x17, lsr#63 + eor x11, x11, x17 + adcs x10, x10, xzr + eor x12, x12, x17 + adcs x11, x11, xzr + eor x13, x13, x17 + adcs x12, x12, xzr + umulh x22, x9, x21 + eor x14, x14, x17 + umulh x23, x10, x21 + adcs x13, x13, xzr + umulh x24, x11, x21 + adcs x14, x14, xzr + umulh x25, x12, x21 + adc x19, xzr, xzr // used in __smul_767x63_tail + umulh x26, x13, x21 + mul x9, x9, x21 + mul x10, x10, x21 + mul x11, x11, x21 + adds x10, x10, x22 + mul x12, x12, x21 + adcs x11, x11, x23 + mul x13, x13, x21 + adcs x12, x12, x24 + mul x28,x14, x21 + adcs x13, x13, x25 + adcs x28,x28,x26 + adc x2, x2, xzr + + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + stp x3, x4, [x0,#8*0] + adcs x7, x7, x13 + stp x5, x6, [x0,#8*2] + adcs x27, x27, x28 + stp x7, x27, [x0,#8*4] + adc x28, x2, xzr // used in __smul_767x63_tail + + ret + ENDP + + + ALIGN 32 +|__smul_767x63_tail| PROC + smulh x27, x8, x20 + ldp x3, x4, [x1,#8*24] // load rest of |v| + umulh x14,x14, x21 + ldp x5, x6, [x1,#8*26] + ldp x7, x8, [x1,#8*28] + + eor x3, x3, x17 // conditionally negate rest of |v| + eor x4, x4, x17 + eor x5, x5, x17 + adds x3, x3, x19 + eor x6, x6, x17 + adcs x4, x4, xzr + eor x7, x7, x17 + adcs x5, x5, xzr + eor x8, x8, x17 + adcs x6, x6, xzr + umulh x22, x3, x21 + adcs x7, x7, xzr + umulh x23, x4, x21 + adc x8, x8, xzr + + umulh x24, x5, x21 + add x14, x14, x28 + umulh x25, x6, x21 + asr x28, x27, #63 + umulh x26, x7, x21 + mul x3, x3, x21 + mul x4, x4, x21 + mul x5, x5, x21 + adds x3, x3, x14 + mul x6, x6, x21 + adcs x4, x4, x22 + mul x7, x7, x21 + adcs x5, x5, x23 + mul x8, x8, x21 + adcs x6, x6, x24 + adcs x7, x7, x25 + adc x8, x8, x26 + + adds x3, x3, x27 + adcs x4, x4, x28 + adcs x5, x5, x28 + adcs x6, x6, x28 + stp x3, x4, [x0,#8*6] + adcs x7, x7, x28 + stp x5, x6, [x0,#8*8] + adc x8, x8, x28 + stp x7, x8, [x0,#8*10] + + ret + ENDP + + + ALIGN 32 +|__smul_383_n_shift_by_62| PROC + ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) + asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x5, x6, [x1,#8*2+0] + eor x2, x15, x28 // conditionally negate |f0| (or |g0|) + ldp x7, x8, [x1,#8*4+0] + + eor x3, x3, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + umulh x22, x3, x2 + adcs x6, x6, xzr + umulh x23, x4, x2 + eor x8, x8, x28 + umulh x24, x5, x2 + adcs x7, x7, xzr + umulh x25, x6, x2 + adc x8, x8, xzr + + umulh x26, x7, x2 + smulh x27, x8, x2 + mul x3, x3, x2 + mul x4, x4, x2 + mul x5, x5, x2 + adds x4, x4, x22 + mul x6, x6, x2 + adcs x5, x5, x23 + mul x7, x7, x2 + adcs x6, x6, x24 + mul x8, x8, x2 + adcs x7, x7, x25 + adcs x8, x8 ,x26 + adc x27, x27, xzr + ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) + asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) + ldp x11, x12, [x1,#8*2+48] + eor x2, x16, x28 // conditionally negate |f0| (or |g0|) + ldp x13, x14, [x1,#8*4+48] + + eor x9, x9, x28 // conditionally negate |a| (or |b|) + sub x2, x2, x28 + eor x10, x10, x28 + adds x9, x9, x28, lsr#63 + eor x11, x11, x28 + adcs x10, x10, xzr + eor x12, x12, x28 + adcs x11, x11, xzr + eor x13, x13, x28 + umulh x22, x9, x2 + adcs x12, x12, xzr + umulh x23, x10, x2 + eor x14, x14, x28 + umulh x24, x11, x2 + adcs x13, x13, xzr + umulh x25, x12, x2 + adc x14, x14, xzr + + umulh x26, x13, x2 + smulh x28, x14, x2 + mul x9, x9, x2 + mul x10, x10, x2 + mul x11, x11, x2 + adds x10, x10, x22 + mul x12, x12, x2 + adcs x11, x11, x23 + mul x13, x13, x2 + adcs x12, x12, x24 + mul x14, x14, x2 + adcs x13, x13, x25 + adcs x14, x14 ,x26 + adc x28, x28, xzr + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x27, x28 + + extr x3, x4, x3, #62 + extr x4, x5, x4, #62 + extr x5, x6, x5, #62 + asr x28, x9, #63 + extr x6, x7, x6, #62 + extr x7, x8, x7, #62 + extr x8, x9, x8, #62 + + eor x3, x3, x28 + eor x4, x4, x28 + adds x3, x3, x28, lsr#63 + eor x5, x5, x28 + adcs x4, x4, xzr + eor x6, x6, x28 + adcs x5, x5, xzr + eor x7, x7, x28 + adcs x6, x6, xzr + eor x8, x8, x28 + stp x3, x4, [x0,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x0,#8*2] + adc x8, x8, xzr + stp x7, x8, [x0,#8*4] + + eor x15, x15, x28 + eor x16, x16, x28 + sub x15, x15, x28 + sub x16, x16, x28 + + ret + ENDP + + ALIGN 16 +|__ab_approximation_62| PROC + ldp x7, x8, [x1,#8*4] + ldp x13, x14, [x1,#8*10] + ldp x5, x6, [x1,#8*2] + ldp x11, x12, [x1,#8*8] + +|$Lab_approximation_62_loaded| + orr x22, x8, x14 // check top-most limbs, ... + cmp x22, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x6 + orr x22, x8, x14 // ... ones before top-most, ... + cselne x13,x13,x12 + + ldp x3, x4, [x1,#8*0] + ldp x9, x10, [x1,#8*6] + + cmp x22, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x5 + orr x22, x8, x14 // ... and ones before that ... + cselne x13,x13,x11 + + cmp x22, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x4 + orr x22, x8, x14 + cselne x13,x13,x10 + + clz x22, x22 + cmp x22, #64 + cselne x22,x22,xzr + cselne x8,x8,x7 + cselne x14,x14,x13 + neg x23, x22 + + lslv x8, x8, x22 // align high limbs to the left + lslv x14, x14, x22 + lsrv x7, x7, x23 + lsrv x13, x13, x23 + and x7, x7, x23, asr#6 + and x13, x13, x23, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + b __inner_loop_62 + ret + ENDP + + ALIGN 16 +|__inner_loop_62| PROC + mov x15, #1 // |f0|=1 + mov x16, #0 // |g0|=0 + mov x17, #0 // |f1|=0 + mov x19, #1 // |g1|=1 + +|$Loop_62| + sbfx x28, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + sub x2, x2, #1 + subs x24, x9, x3 // |b_|-|a_| + and x22, x9, x28 + sbc x25, x14, x8 + and x23, x14, x28 + subs x26, x3, x22 // |a_|-|b_| (or |a_|-0 if |a_| was even) + mov x22, x15 + sbcs x27, x8, x23 + mov x23, x16 + cselhs x9,x9,x3 + cselhs x14,x14,x8 + cselhs x3,x26,x24 + cselhs x8,x27,x25 + cselhs x15,x15,x17 + cselhs x17,x17,x22 + cselhs x16,x16,x19 + cselhs x19,x19,x23 + extr x3, x8, x3, #1 + lsr x8, x8, #1 + and x22, x17, x28 + and x23, x19, x28 + add x17, x17, x17 // |f1|<<=1 + add x19, x19, x19 // |g1|<<=1 + sub x15, x15, x22 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + sub x16, x16, x23 // |g0|-=|g1| (or |g0-=0| ...) + cbnz x2, |$Loop_62| + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/ct_is_square_mod_384-armv8.asm b/crypto/blst_src/build/win64/ct_is_square_mod_384-armv8.asm new file mode 100644 index 00000000000..ab72328f056 --- /dev/null +++ b/crypto/blst_src/build/win64/ct_is_square_mod_384-armv8.asm @@ -0,0 +1,325 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + EXPORT |ct_is_square_mod_384|[FUNC] + ALIGN 32 +|ct_is_square_mod_384| PROC + DCDU 3573752639 + stp x29, x30, [sp,#-128]! + add x29, sp, #0 + stp x19, x20, [sp,#16] + stp x21, x22, [sp,#32] + stp x23, x24, [sp,#48] + stp x25, x26, [sp,#64] + stp x27, x28, [sp,#80] + sub sp, sp, #512 + + ldp x3, x4, [x0,#8*0] // load input + ldp x5, x6, [x0,#8*2] + ldp x7, x8, [x0,#8*4] + + add x0, sp, #255 // find closest 256-byte-aligned spot + and x0, x0, #-256 // in the frame... + + ldp x9, x10, [x1,#8*0] // load modulus + ldp x11, x12, [x1,#8*2] + ldp x13, x14, [x1,#8*4] + + stp x3, x4, [x0,#8*6] // copy input to |a| + stp x5, x6, [x0,#8*8] + stp x7, x8, [x0,#8*10] + stp x9, x10, [x0,#8*0] // copy modulus to |b| + stp x11, x12, [x0,#8*2] + stp x13, x14, [x0,#8*4] + + eor x2, x2, x2 // init the |$Legendre| symbol + mov x15, #24 // 24 is 768/30-1 + b |$Loop_is_square| + + ALIGN 16 +|$Loop_is_square| + bl __ab_approximation_30 + sub x15, x15, #1 + + eor x1, x0, #128 // pointer to dst |b| + bl __smul_384_n_shift_by_30 + + mov x19, x16 // |f0| + mov x20, x17 // |g0| + add x1, x1, #8*6 // pointer to dst |a| + bl __smul_384_n_shift_by_30 + + ldp x9, x10, [x1,#-8*6] + eor x0, x0, #128 // flip-flop src |a|b| + and x27, x27, x9 // if |a| was negative, + add x2, x2, x27, lsr#1 // adjust |L| + + cbnz x15, |$Loop_is_square| + + ////////////////////////////////////////// last iteration + //bl __ab_approximation_30 // |a| and |b| are exact, + //ldr x8, [x0,#8*6] // and loaded + //ldr x14, [x0,#8*0] + mov x15, #48 // 48 is 768%30 + 30 + bl __inner_loop_48 + ldr x30, [x29,#8] + + and x0, x2, #1 + eor x0, x0, #1 + + add sp, sp, #512 + ldp x19, x20, [x29,#16] + ldp x21, x22, [x29,#32] + ldp x23, x24, [x29,#48] + ldp x25, x26, [x29,#64] + ldp x27, x28, [x29,#80] + ldr x29, [sp],#128 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__smul_384_n_shift_by_30| PROC + ldp x3, x4, [x0,#8*0+0] // load |b| (or |a|) + asr x27, x20, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x5, x6, [x0,#8*2+0] + eor x20, x20, x27 // conditionally negate |g1| (or |f1|) + ldp x7, x8, [x0,#8*4+0] + + eor x3, x3, x27 // conditionally negate |b| (or |a|) + sub x20, x20, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + umulh x21, x3, x20 + adcs x6, x6, xzr + umulh x22, x4, x20 + eor x8, x8, x27 + umulh x23, x5, x20 + adcs x7, x7, xzr + umulh x24, x6, x20 + adc x8, x8, xzr + + umulh x25, x7, x20 + and x28, x20, x27 + umulh x26, x8, x20 + neg x28, x28 + mul x3, x3, x20 + mul x4, x4, x20 + mul x5, x5, x20 + adds x4, x4, x21 + mul x6, x6, x20 + adcs x5, x5, x22 + mul x7, x7, x20 + adcs x6, x6, x23 + mul x8, x8, x20 + adcs x7, x7, x24 + adcs x8, x8 ,x25 + adc x26, x26, x28 + ldp x9, x10, [x0,#8*0+48] // load |b| (or |a|) + asr x27, x19, #63 // |g1|'s sign as mask (or |f1|'s) + ldp x11, x12, [x0,#8*2+48] + eor x19, x19, x27 // conditionally negate |g1| (or |f1|) + ldp x13, x14, [x0,#8*4+48] + + eor x9, x9, x27 // conditionally negate |b| (or |a|) + sub x19, x19, x27 + eor x10, x10, x27 + adds x9, x9, x27, lsr#63 + eor x11, x11, x27 + adcs x10, x10, xzr + eor x12, x12, x27 + adcs x11, x11, xzr + eor x13, x13, x27 + umulh x21, x9, x19 + adcs x12, x12, xzr + umulh x22, x10, x19 + eor x14, x14, x27 + umulh x23, x11, x19 + adcs x13, x13, xzr + umulh x24, x12, x19 + adc x14, x14, xzr + + umulh x25, x13, x19 + and x28, x19, x27 + umulh x27, x14, x19 + neg x28, x28 + mul x9, x9, x19 + mul x10, x10, x19 + mul x11, x11, x19 + adds x10, x10, x21 + mul x12, x12, x19 + adcs x11, x11, x22 + mul x13, x13, x19 + adcs x12, x12, x23 + mul x14, x14, x19 + adcs x13, x13, x24 + adcs x14, x14 ,x25 + adc x27, x27, x28 + adds x3, x3, x9 + adcs x4, x4, x10 + adcs x5, x5, x11 + adcs x6, x6, x12 + adcs x7, x7, x13 + adcs x8, x8, x14 + adc x9, x26, x27 + + extr x3, x4, x3, #30 + extr x4, x5, x4, #30 + extr x5, x6, x5, #30 + asr x27, x9, #63 + extr x6, x7, x6, #30 + extr x7, x8, x7, #30 + extr x8, x9, x8, #30 + + eor x3, x3, x27 + eor x4, x4, x27 + adds x3, x3, x27, lsr#63 + eor x5, x5, x27 + adcs x4, x4, xzr + eor x6, x6, x27 + adcs x5, x5, xzr + eor x7, x7, x27 + adcs x6, x6, xzr + eor x8, x8, x27 + stp x3, x4, [x1,#8*0] + adcs x7, x7, xzr + stp x5, x6, [x1,#8*2] + adc x8, x8, xzr + stp x7, x8, [x1,#8*4] + + ret + ENDP + + ALIGN 16 +|__ab_approximation_30| PROC + ldp x13, x14, [x0,#8*4] // |a| is still in registers + ldp x11, x12, [x0,#8*2] + + orr x21, x8, x14 // check top-most limbs, ... + cmp x21, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x6 + orr x21, x8, x14 // ... ones before top-most, ... + cselne x13,x13,x12 + + cmp x21, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x5 + orr x21, x8, x14 // ... and ones before that ... + cselne x13,x13,x11 + + cmp x21, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x4 + orr x21, x8, x14 // and one more, ... + cselne x13,x13,x10 + + cmp x21, #0 + cselne x8,x8,x7 + cselne x14,x14,x13 + cselne x7,x7,x3 + orr x21, x8, x14 + cselne x13,x13,x9 + + clz x21, x21 + cmp x21, #64 + cselne x21,x21,xzr + cselne x8,x8,x7 + cselne x14,x14,x13 + neg x22, x21 + + lslv x8, x8, x21 // align high limbs to the left + lslv x14, x14, x21 + lsrv x7, x7, x22 + lsrv x13, x13, x22 + and x7, x7, x22, asr#6 + and x13, x13, x22, asr#6 + orr x8, x8, x7 + orr x14, x14, x13 + + bfxil x8, x3, #0, #32 + bfxil x14, x9, #0, #32 + + b __inner_loop_30 + ret + ENDP + + + ALIGN 16 +|__inner_loop_30| PROC + mov x28, #30 + mov x17, #0x7FFFFFFF80000000 // |f0|=1, |g0|=0 + mov x20, #0x800000007FFFFFFF // |f1|=0, |g1|=1 + mov x27,#0x7FFFFFFF7FFFFFFF + +|$Loop_30| + sbfx x24, x8, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x8, x14 + sub x28, x28, #1 + and x21, x14, x24 + + sub x22, x14, x8 // |b_|-|a_| + subs x23, x8, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 // L + (a_ & b_) >> 1 + mov x21, x20 + cselhs x14,x14,x8 + cselhs x8,x23,x22 + cselhs x20,x20,x17 + cselhs x17,x17,x21 + cselhs x2,x2,x25 + lsr x8, x8, #1 + and x21, x20, x24 + and x22, x27, x24 + add x23, x14, #2 + sub x17, x17, x21 // |f0|-=|f1| (or |f0-=0| if |a_| was even) + add x20, x20, x20 // |f1|<<=1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + add x17, x17, x22 + sub x20, x20, x27 + + cbnz x28, |$Loop_30| + + mov x27, #0x7FFFFFFF + ubfx x16, x17, #0, #32 + ubfx x17, x17, #32, #32 + ubfx x19, x20, #0, #32 + ubfx x20, x20, #32, #32 + sub x16, x16, x27 // remove the bias + sub x17, x17, x27 + sub x19, x19, x27 + sub x20, x20, x27 + + ret + ENDP + + ALIGN 16 +|__inner_loop_48| PROC +|$Loop_48| + sbfx x24, x3, #0, #1 // if |a_| is odd, then we'll be subtracting + and x25, x3, x9 + sub x15, x15, #1 + and x21, x9, x24 + sub x22, x9, x3 // |b_|-|a_| + subs x23, x3, x21 // |a_|-|b_| (or |a_|-0 if |a_| was even) + add x25, x2, x25, lsr#1 + cselhs x9,x9,x3 + cselhs x3,x23,x22 + cselhs x2,x2,x25 + add x23, x9, #2 + lsr x3, x3, #1 + add x2, x2, x23, lsr#2 // "negate" |L| if |b|%8 is 3 or 5 + + cbnz x15, |$Loop_48| + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/ct_is_square_mod_384-x86_64.asm b/crypto/blst_src/build/win64/ct_is_square_mod_384-x86_64.asm new file mode 100644 index 00000000000..38de6fc1229 --- /dev/null +++ b/crypto/blst_src/build/win64/ct_is_square_mod_384-x86_64.asm @@ -0,0 +1,509 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC ct_is_square_mod_384 + +ALIGN 32 +ct_is_square_mod_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_ct_is_square_mod_384:: + mov rdi,rcx + mov rsi,rdx + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,536 + +$L$SEH_body_ct_is_square_mod_384:: + + + lea rax,QWORD PTR[((24+255))+rsp] + and rax,-256 + + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov r12,QWORD PTR[32+rdi] + mov r13,QWORD PTR[40+rdi] + + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rbx,QWORD PTR[16+rsi] + mov rcx,QWORD PTR[24+rsi] + mov rdx,QWORD PTR[32+rsi] + mov rdi,QWORD PTR[40+rsi] + mov rsi,rax + + mov QWORD PTR[rax],r8 + mov QWORD PTR[8+rax],r9 + mov QWORD PTR[16+rax],r10 + mov QWORD PTR[24+rax],r11 + mov QWORD PTR[32+rax],r12 + mov QWORD PTR[40+rax],r13 + + mov QWORD PTR[48+rax],r14 + mov QWORD PTR[56+rax],r15 + mov QWORD PTR[64+rax],rbx + mov QWORD PTR[72+rax],rcx + mov QWORD PTR[80+rax],rdx + mov QWORD PTR[88+rax],rdi + + xor rbp,rbp + mov ecx,24 + jmp $L$oop_is_square + +ALIGN 32 +$L$oop_is_square:: + mov DWORD PTR[16+rsp],ecx + + call __ab_approximation_30 + mov QWORD PTR[rsp],rax + mov QWORD PTR[8+rsp],rbx + + mov rdi,128+8*6 + xor rdi,rsi + call __smulq_384_n_shift_by_30 + + mov rdx,QWORD PTR[rsp] + mov rcx,QWORD PTR[8+rsp] + lea rdi,QWORD PTR[((-48))+rdi] + call __smulq_384_n_shift_by_30 + + mov ecx,DWORD PTR[16+rsp] + xor rsi,128 + + and r14,QWORD PTR[48+rdi] + shr r14,1 + add rbp,r14 + + sub ecx,1 + jnz $L$oop_is_square + + + + + mov r9,QWORD PTR[48+rsi] + call __inner_loop_48 + + mov rax,1 + and rax,rbp + xor rax,1 + + lea r8,QWORD PTR[536+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_ct_is_square_mod_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_ct_is_square_mod_384:: +ct_is_square_mod_384 ENDP + + +ALIGN 32 +__smulq_384_n_shift_by_30 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbx,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbx,rdx + add rbx,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mov r14,rdx + and r14,rbx + mul rbx + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbx + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbx + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbx + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbx + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + neg r14 + mul rbx + add r13,rax + adc r14,rdx + lea rsi,QWORD PTR[48+rsi] + mov rdx,rcx + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbx,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbx,rdx + add rbx,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mov r15,rdx + and r15,rbx + mul rbx + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbx + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbx + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbx + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbx + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + neg r15 + mul rbx + add r13,rax + adc r15,rdx + lea rsi,QWORD PTR[((-48))+rsi] + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + adc r14,r15 + + shrd r8,r9,30 + shrd r9,r10,30 + shrd r10,r11,30 + shrd r11,r12,30 + shrd r12,r13,30 + shrd r13,r14,30 + + sar r14,63 + xor rbx,rbx + sub rbx,r14 + + xor r8,r14 + xor r9,r14 + xor r10,r14 + xor r11,r14 + xor r12,r14 + xor r13,r14 + add r8,rbx + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__smulq_384_n_shift_by_30 ENDP + +ALIGN 32 +__ab_approximation_30 PROC PRIVATE + DB 243,15,30,250 + mov rbx,QWORD PTR[88+rsi] + mov r15,QWORD PTR[80+rsi] + mov r14,QWORD PTR[72+rsi] + + mov rax,r13 + or rax,rbx + cmovz r13,r12 + cmovz rbx,r15 + cmovz r12,r11 + mov r11,QWORD PTR[64+rsi] + cmovz r15,r14 + + mov rax,r13 + or rax,rbx + cmovz r13,r12 + cmovz rbx,r15 + cmovz r12,r10 + mov r10,QWORD PTR[56+rsi] + cmovz r15,r11 + + mov rax,r13 + or rax,rbx + cmovz r13,r12 + cmovz rbx,r15 + cmovz r12,r9 + mov r9,QWORD PTR[48+rsi] + cmovz r15,r10 + + mov rax,r13 + or rax,rbx + cmovz r13,r12 + cmovz rbx,r15 + cmovz r12,r8 + cmovz r15,r9 + + mov rax,r13 + or rax,rbx + bsr rcx,rax + lea rcx,QWORD PTR[1+rcx] + cmovz r13,r8 + cmovz rbx,r9 + cmovz rcx,rax + neg rcx + + + shld r13,r12,cl + shld rbx,r15,cl + + mov rax,0FFFFFFFF00000000h + mov r8d,r8d + mov r9d,r9d + and r13,rax + and rbx,rax + or r8,r13 + or r9,rbx + + jmp __inner_loop_30 + + DB 0F3h,0C3h ;repret +__ab_approximation_30 ENDP + +ALIGN 32 +__inner_loop_30 PROC PRIVATE + DB 243,15,30,250 + mov rbx,07FFFFFFF80000000h + mov rcx,0800000007FFFFFFFh + lea r15,QWORD PTR[((-1))+rbx] + mov edi,30 + +$L$oop_30:: + mov rax,r8 + and rax,r9 + shr rax,1 + + cmp r8,r9 + mov r10,r8 + mov r11,r9 + lea rax,QWORD PTR[rbp*1+rax] + mov r12,rbx + mov r13,rcx + mov r14,rbp + cmovb r8,r9 + cmovb r9,r10 + cmovb rbx,rcx + cmovb rcx,r12 + cmovb rbp,rax + + sub r8,r9 + sub rbx,rcx + add rbx,r15 + + test r10,1 + cmovz r8,r10 + cmovz r9,r11 + cmovz rbx,r12 + cmovz rcx,r13 + cmovz rbp,r14 + + lea rax,QWORD PTR[2+r9] + shr r8,1 + shr rax,2 + add rcx,rcx + lea rbp,QWORD PTR[rbp*1+rax] + sub rcx,r15 + + sub edi,1 + jnz $L$oop_30 + + shr r15,32 + mov eax,ebx + shr rbx,32 + mov edx,ecx + shr rcx,32 + sub rax,r15 + sub rbx,r15 + sub rdx,r15 + sub rcx,r15 + + DB 0F3h,0C3h ;repret +__inner_loop_30 ENDP + + +ALIGN 32 +__inner_loop_48 PROC PRIVATE + DB 243,15,30,250 + mov edi,48 + +$L$oop_48:: + mov rax,r8 + and rax,r9 + shr rax,1 + + cmp r8,r9 + mov r10,r8 + mov r11,r9 + lea rax,QWORD PTR[rbp*1+rax] + mov r12,rbp + cmovb r8,r9 + cmovb r9,r10 + cmovb rbp,rax + + sub r8,r9 + + test r10,1 + cmovz r8,r10 + cmovz r9,r11 + cmovz rbp,r12 + + lea rax,QWORD PTR[2+r9] + shr r8,1 + shr rax,2 + add rbp,rax + + sub edi,1 + jnz $L$oop_48 + + DB 0F3h,0C3h ;repret +__inner_loop_48 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_ct_is_square_mod_384 + DD imagerel $L$SEH_body_ct_is_square_mod_384 + DD imagerel $L$SEH_info_ct_is_square_mod_384_prologue + + DD imagerel $L$SEH_body_ct_is_square_mod_384 + DD imagerel $L$SEH_epilogue_ct_is_square_mod_384 + DD imagerel $L$SEH_info_ct_is_square_mod_384_body + + DD imagerel $L$SEH_epilogue_ct_is_square_mod_384 + DD imagerel $L$SEH_end_ct_is_square_mod_384 + DD imagerel $L$SEH_info_ct_is_square_mod_384_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_ct_is_square_mod_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_ct_is_square_mod_384_body:: +DB 1,0,18,0 +DB 000h,0f4h,043h,000h +DB 000h,0e4h,044h,000h +DB 000h,0d4h,045h,000h +DB 000h,0c4h,046h,000h +DB 000h,034h,047h,000h +DB 000h,054h,048h,000h +DB 000h,074h,04ah,000h +DB 000h,064h,04bh,000h +DB 000h,001h,049h,000h +$L$SEH_info_ct_is_square_mod_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm b/crypto/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm new file mode 100644 index 00000000000..de79f8ec80e --- /dev/null +++ b/crypto/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm @@ -0,0 +1,1224 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC ct_inverse_mod_383 + +ALIGN 32 +ct_inverse_mod_383 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_ct_inverse_mod_383:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,1112 + +$L$SEH_body_ct_inverse_mod_383:: + + + lea rax,QWORD PTR[((88+511))+rsp] + and rax,-512 + mov QWORD PTR[32+rsp],rdi + mov QWORD PTR[40+rsp],rcx + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,QWORD PTR[rdx] + mov r15,QWORD PTR[8+rdx] + mov rbx,QWORD PTR[16+rdx] + mov rbp,QWORD PTR[24+rdx] + mov rsi,QWORD PTR[32+rdx] + mov rdi,QWORD PTR[40+rdx] + + mov QWORD PTR[rax],r8 + mov QWORD PTR[8+rax],r9 + mov QWORD PTR[16+rax],r10 + mov QWORD PTR[24+rax],r11 + mov QWORD PTR[32+rax],r12 + mov QWORD PTR[40+rax],r13 + + mov QWORD PTR[48+rax],r14 + mov QWORD PTR[56+rax],r15 + mov QWORD PTR[64+rax],rbx + mov QWORD PTR[72+rax],rbp + mov QWORD PTR[80+rax],rsi + mov rsi,rax + mov QWORD PTR[88+rax],rdi + + + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + + + mov QWORD PTR[96+rdi],rdx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + + + mov QWORD PTR[96+rdi],rdx + + + xor rsi,256 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + + + + mov rax,QWORD PTR[96+rsi] + mov r11,QWORD PTR[144+rsi] + mov rbx,rdx + mov r10,rax + imul QWORD PTR[56+rsp] + mov r8,rax + mov rax,r11 + mov r9,rdx + imul QWORD PTR[64+rsp] + add r8,rax + adc r9,rdx + mov QWORD PTR[48+rdi],r8 + mov QWORD PTR[56+rdi],r9 + sar r9,63 + mov QWORD PTR[64+rdi],r9 + mov QWORD PTR[72+rdi],r9 + mov QWORD PTR[80+rdi],r9 + mov QWORD PTR[88+rdi],r9 + lea rsi,QWORD PTR[96+rsi] + + mov rax,r10 + imul rbx + mov r8,rax + mov rax,r11 + mov r9,rdx + imul rcx + add r8,rax + adc r9,rdx + mov QWORD PTR[96+rdi],r8 + mov QWORD PTR[104+rdi],r9 + sar r9,63 + mov QWORD PTR[112+rdi],r9 + mov QWORD PTR[120+rdi],r9 + mov QWORD PTR[128+rdi],r9 + mov QWORD PTR[136+rdi],r9 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + sar r13,63 + mov QWORD PTR[48+rdi],r13 + mov QWORD PTR[56+rdi],r13 + mov QWORD PTR[64+rdi],r13 + mov QWORD PTR[72+rdi],r13 + mov QWORD PTR[80+rdi],r13 + mov QWORD PTR[88+rdi],r13 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + xor rsi,256+8*12 + mov edi,62 + call __ab_approximation_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulq_383_n_shift_by_62 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383_n_shift_by_62 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + + xor rsi,256+8*12 + mov edi,62 + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[48+rsi] + mov r11,QWORD PTR[56+rsi] + call __inner_loop_62 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + mov QWORD PTR[rdi],r8 + mov QWORD PTR[48+rdi],r10 + + + + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[96+rdi] + call __smulq_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulq_767x63 + + + xor rsi,256+8*12 + mov edi,22 + + mov r8,QWORD PTR[rsi] + xor r9,r9 + mov r10,QWORD PTR[48+rsi] + xor r11,r11 + call __inner_loop_62 + + + + + + + + lea rsi,QWORD PTR[96+rsi] + + + + + + mov rdx,r12 + mov rcx,r13 + mov rdi,QWORD PTR[32+rsp] + call __smulq_767x63 + + mov rsi,QWORD PTR[40+rsp] + mov rdx,rax + sar rax,63 + + mov r8,rax + mov r9,rax + mov r10,rax + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,rax + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,rax + and r12,QWORD PTR[32+rsi] + and rax,QWORD PTR[40+rsi] + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rdx,rax + + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rdx + + lea r8,QWORD PTR[1112+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_ct_inverse_mod_383:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_ct_inverse_mod_383:: +ct_inverse_mod_383 ENDP + +ALIGN 32 +__smulq_767x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbp,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + mov QWORD PTR[8+rsp],rdi + mov QWORD PTR[16+rsp],rsi + lea rsi,QWORD PTR[48+rsi] + + xor rbp,rdx + add rbp,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mul rbp + mov QWORD PTR[rdi],rax + mov rax,r9 + mov r9,rdx + mul rbp + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mov QWORD PTR[8+rdi],r9 + mul rbp + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mov QWORD PTR[16+rdi],r10 + mul rbp + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mov QWORD PTR[24+rdi],r11 + mul rbp + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + mov QWORD PTR[32+rdi],r12 + imul rbp + add r13,rax + adc rdx,0 + + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],rdx + sar rdx,63 + mov QWORD PTR[56+rdi],rdx + mov rdx,rcx + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + mov r15,QWORD PTR[56+rsi] + mov rbx,QWORD PTR[64+rsi] + mov rbp,QWORD PTR[72+rsi] + mov rcx,QWORD PTR[80+rsi] + mov rdi,QWORD PTR[88+rsi] + + mov rsi,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rsi,rdx + add rsi,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + xor r14,rdx + xor r15,rdx + xor rbx,rdx + xor rbp,rdx + xor rcx,rdx + xor rdi,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + adc r15,0 + adc rbx,0 + adc rbp,0 + adc rcx,0 + adc rdi,0 + + mul rsi + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rsi + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rsi + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rsi + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rsi + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + mul rsi + add r13,rax + mov rax,r14 + adc rdx,0 + mov r14,rdx + mul rsi + add r14,rax + mov rax,r15 + adc rdx,0 + mov r15,rdx + mul rsi + add r15,rax + mov rax,rbx + adc rdx,0 + mov rbx,rdx + mul rsi + add rbx,rax + mov rax,rbp + adc rdx,0 + mov rbp,rdx + mul rsi + add rbp,rax + mov rax,rcx + adc rdx,0 + mov rcx,rdx + mul rsi + add rcx,rax + mov rax,rdi + adc rdx,0 + mov rdi,rdx + mov rdx,QWORD PTR[8+rsp] + imul rax,rsi + mov rsi,QWORD PTR[16+rsp] + add rax,rdi + + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + adc r10,QWORD PTR[16+rdx] + adc r11,QWORD PTR[24+rdx] + adc r12,QWORD PTR[32+rdx] + adc r13,QWORD PTR[40+rdx] + adc r14,QWORD PTR[48+rdx] + mov rdi,QWORD PTR[56+rdx] + adc r15,rdi + adc rbx,rdi + adc rbp,rdi + adc rcx,rdi + adc rax,rdi + + mov rdi,rdx + + mov QWORD PTR[rdx],r8 + mov QWORD PTR[8+rdx],r9 + mov QWORD PTR[16+rdx],r10 + mov QWORD PTR[24+rdx],r11 + mov QWORD PTR[32+rdx],r12 + mov QWORD PTR[40+rdx],r13 + mov QWORD PTR[48+rdx],r14 + mov QWORD PTR[56+rdx],r15 + mov QWORD PTR[64+rdx],rbx + mov QWORD PTR[72+rdx],rbp + mov QWORD PTR[80+rdx],rcx + mov QWORD PTR[88+rdx],rax + + DB 0F3h,0C3h ;repret +__smulq_767x63 ENDP + +ALIGN 32 +__smulq_383x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbp,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbp,rdx + add rbp,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mul rbp + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbp + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbp + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbp + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbp + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + imul rax,rbp + add r13,rax + + lea rsi,QWORD PTR[48+rsi] + mov rdx,rcx + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbp,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbp,rdx + add rbp,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mul rbp + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbp + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbp + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbp + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbp + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + imul rax,rbp + add r13,rax + + lea rsi,QWORD PTR[((-48))+rsi] + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__smulq_383x63 ENDP + +ALIGN 32 +__smulq_383_n_shift_by_62 PROC PRIVATE + DB 243,15,30,250 + mov rbx,rdx + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbp,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbp,rdx + add rbp,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mul rbp + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbp + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbp + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbp + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbp + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + imul rbp + add r13,rax + adc rdx,0 + + lea rsi,QWORD PTR[48+rsi] + mov r14,rdx + mov rdx,rcx + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rbp,rdx + sar rdx,63 + xor rax,rax + sub rax,rdx + + xor rbp,rdx + add rbp,rax + + xor r8,rdx + xor r9,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add rax,r8 + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mul rbp + mov r8,rax + mov rax,r9 + mov r9,rdx + mul rbp + add r9,rax + mov rax,r10 + adc rdx,0 + mov r10,rdx + mul rbp + add r10,rax + mov rax,r11 + adc rdx,0 + mov r11,rdx + mul rbp + add r11,rax + mov rax,r12 + adc rdx,0 + mov r12,rdx + mul rbp + add r12,rax + mov rax,r13 + adc rdx,0 + mov r13,rdx + imul rbp + add r13,rax + adc rdx,0 + + lea rsi,QWORD PTR[((-48))+rsi] + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + adc r14,rdx + mov rdx,rbx + + shrd r8,r9,62 + shrd r9,r10,62 + shrd r10,r11,62 + shrd r11,r12,62 + shrd r12,r13,62 + shrd r13,r14,62 + + sar r14,63 + xor rbp,rbp + sub rbp,r14 + + xor r8,r14 + xor r9,r14 + xor r10,r14 + xor r11,r14 + xor r12,r14 + xor r13,r14 + add r8,rbp + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + xor rdx,r14 + xor rcx,r14 + add rdx,rbp + add rcx,rbp + + DB 0F3h,0C3h ;repret +__smulq_383_n_shift_by_62 ENDP + +ALIGN 32 +__ab_approximation_62 PROC PRIVATE + DB 243,15,30,250 + mov r9,QWORD PTR[40+rsi] + mov r11,QWORD PTR[88+rsi] + mov rbx,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[80+rsi] + mov r8,QWORD PTR[24+rsi] + mov r10,QWORD PTR[72+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + cmovz rbp,r10 + mov r8,QWORD PTR[16+rsi] + mov r10,QWORD PTR[64+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + cmovz rbp,r10 + mov r8,QWORD PTR[8+rsi] + mov r10,QWORD PTR[56+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + cmovz rbp,r10 + mov r8,QWORD PTR[rsi] + mov r10,QWORD PTR[48+rsi] + + mov rax,r9 + or rax,r11 + bsr rcx,rax + lea rcx,QWORD PTR[1+rcx] + cmovz r9,rbx + cmovz r11,rbp + cmovz rcx,rax + neg rcx + + + shld r9,rbx,cl + shld r11,rbp,cl + + jmp __inner_loop_62 + + DB 0F3h,0C3h ;repret +__ab_approximation_62 ENDP + +ALIGN 8 + DD 0 +__inner_loop_62 PROC PRIVATE + DB 243,15,30,250 + mov rdx,1 + xor rcx,rcx + xor r12,r12 + mov r13,1 + mov QWORD PTR[8+rsp],rsi + +$L$oop_62:: + xor rax,rax + xor rbx,rbx + test r8,1 + mov rbp,r10 + mov r14,r11 + cmovnz rax,r10 + cmovnz rbx,r11 + sub rbp,r8 + sbb r14,r9 + mov r15,r8 + mov rsi,r9 + sub r8,rax + sbb r9,rbx + cmovc r8,rbp + cmovc r9,r14 + cmovc r10,r15 + cmovc r11,rsi + mov rax,rdx + cmovc rdx,r12 + cmovc r12,rax + mov rbx,rcx + cmovc rcx,r13 + cmovc r13,rbx + xor rax,rax + xor rbx,rbx + shrd r8,r9,1 + shr r9,1 + test r15,1 + cmovnz rax,r12 + cmovnz rbx,r13 + add r12,r12 + add r13,r13 + sub rdx,rax + sub rcx,rbx + sub edi,1 + jnz $L$oop_62 + + mov rsi,QWORD PTR[8+rsp] + DB 0F3h,0C3h ;repret +__inner_loop_62 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_ct_inverse_mod_383 + DD imagerel $L$SEH_body_ct_inverse_mod_383 + DD imagerel $L$SEH_info_ct_inverse_mod_383_prologue + + DD imagerel $L$SEH_body_ct_inverse_mod_383 + DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 + DD imagerel $L$SEH_info_ct_inverse_mod_383_body + + DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 + DD imagerel $L$SEH_end_ct_inverse_mod_383 + DD imagerel $L$SEH_info_ct_inverse_mod_383_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_ct_inverse_mod_383_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_ct_inverse_mod_383_body:: +DB 1,0,18,0 +DB 000h,0f4h,08bh,000h +DB 000h,0e4h,08ch,000h +DB 000h,0d4h,08dh,000h +DB 000h,0c4h,08eh,000h +DB 000h,034h,08fh,000h +DB 000h,054h,090h,000h +DB 000h,074h,092h,000h +DB 000h,064h,093h,000h +DB 000h,001h,091h,000h +$L$SEH_info_ct_inverse_mod_383_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm b/crypto/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm new file mode 100644 index 00000000000..df4c46a4c44 --- /dev/null +++ b/crypto/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm @@ -0,0 +1,1597 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC ctx_inverse_mod_383 + +ALIGN 32 +ctx_inverse_mod_383 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_ctx_inverse_mod_383:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,1112 + +$L$SEH_body_ctx_inverse_mod_383:: + + + lea rax,QWORD PTR[((88+511))+rsp] + and rax,-512 + mov QWORD PTR[32+rsp],rdi + mov QWORD PTR[40+rsp],rcx + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,QWORD PTR[rdx] + mov r15,QWORD PTR[8+rdx] + mov rbx,QWORD PTR[16+rdx] + mov rbp,QWORD PTR[24+rdx] + mov rsi,QWORD PTR[32+rdx] + mov rdi,QWORD PTR[40+rdx] + + mov QWORD PTR[rax],r8 + mov QWORD PTR[8+rax],r9 + mov QWORD PTR[16+rax],r10 + mov QWORD PTR[24+rax],r11 + mov QWORD PTR[32+rax],r12 + mov QWORD PTR[40+rax],r13 + + mov QWORD PTR[48+rax],r14 + mov QWORD PTR[56+rax],r15 + mov QWORD PTR[64+rax],rbx + mov QWORD PTR[72+rax],rbp + mov QWORD PTR[80+rax],rsi + mov rsi,rax + mov QWORD PTR[88+rax],rdi + + + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + + + mov QWORD PTR[96+rdi],rdx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + + + mov QWORD PTR[96+rdi],rdx + + + xor rsi,256 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + + + + mov rax,QWORD PTR[96+rsi] + mov r11,QWORD PTR[144+rsi] + mov rbx,rdx + mov r10,rax + imul QWORD PTR[56+rsp] + mov r8,rax + mov rax,r11 + mov r9,rdx + imul QWORD PTR[64+rsp] + add r8,rax + adc r9,rdx + mov QWORD PTR[48+rdi],r8 + mov QWORD PTR[56+rdi],r9 + sar r9,63 + mov QWORD PTR[64+rdi],r9 + mov QWORD PTR[72+rdi],r9 + mov QWORD PTR[80+rdi],r9 + mov QWORD PTR[88+rdi],r9 + lea rsi,QWORD PTR[96+rsi] + + mov rax,r10 + imul rbx + mov r8,rax + mov rax,r11 + mov r9,rdx + imul rcx + add r8,rax + adc r9,rdx + mov QWORD PTR[96+rdi],r8 + mov QWORD PTR[104+rdi],r9 + sar r9,63 + mov QWORD PTR[112+rdi],r9 + mov QWORD PTR[120+rdi],r9 + mov QWORD PTR[128+rdi],r9 + mov QWORD PTR[136+rdi],r9 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + sar r13,63 + mov QWORD PTR[48+rdi],r13 + mov QWORD PTR[56+rdi],r13 + mov QWORD PTR[64+rdi],r13 + mov QWORD PTR[72+rdi],r13 + mov QWORD PTR[80+rdi],r13 + mov QWORD PTR[88+rdi],r13 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_383_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_191_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_191_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_191_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_191_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_191_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_191_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + xor rsi,256+8*12 + mov edi,31 + call __ab_approximation_31 + + + mov QWORD PTR[72+rsp],r12 + mov QWORD PTR[80+rsp],r13 + + mov rdi,256 + xor rdi,rsi + call __smulx_191_n_shift_by_31 + mov QWORD PTR[56+rsp],rdx + mov QWORD PTR[64+rsp],rcx + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_191_n_shift_by_31 + mov QWORD PTR[72+rsp],rdx + mov QWORD PTR[80+rsp],rcx + + mov rdx,QWORD PTR[56+rsp] + mov rcx,QWORD PTR[64+rsp] + lea rsi,QWORD PTR[96+rsi] + lea rdi,QWORD PTR[48+rdi] + call __smulx_383x63 + + mov rdx,QWORD PTR[72+rsp] + mov rcx,QWORD PTR[80+rsp] + lea rdi,QWORD PTR[48+rdi] + call __smulx_767x63 + + xor rsi,256+8*12 + mov edi,53 + + mov r8,QWORD PTR[rsi] + + mov r10,QWORD PTR[48+rsi] + + call __inner_loop_62 + + + + + + + + lea rsi,QWORD PTR[96+rsi] + + + + + + mov rdx,r12 + mov rcx,r13 + mov rdi,QWORD PTR[32+rsp] + call __smulx_767x63 + + mov rsi,QWORD PTR[40+rsp] + mov rdx,rax + sar rax,63 + + mov r8,rax + mov r9,rax + mov r10,rax + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,rax + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,rax + and r12,QWORD PTR[32+rsi] + and rax,QWORD PTR[40+rsi] + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rdx,rax + + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rdx + + lea r8,QWORD PTR[1112+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_ctx_inverse_mod_383:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_ctx_inverse_mod_383:: +ctx_inverse_mod_383 ENDP + +ALIGN 32 +__smulx_767x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov rax,rdx + sar rax,63 + xor rbp,rbp + sub rbp,rax + + mov QWORD PTR[8+rsp],rdi + mov QWORD PTR[16+rsp],rsi + lea rsi,QWORD PTR[48+rsi] + + xor rdx,rax + add rdx,rbp + + xor r8,rax + xor r9,rax + xor r10,rax + xor r11,rax + xor r12,rax + xor rax,r13 + add r8,rbp + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc rax,0 + + mulx rbp,r8,r8 + mulx r13,r9,r9 + add r9,rbp + mulx rbp,r10,r10 + adc r10,r13 + mulx r13,r11,r11 + adc r11,rbp + mulx rbp,r12,r12 + adc r12,r13 + adc rbp,0 + imul rdx + add rax,rbp + adc rdx,0 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],rax + mov QWORD PTR[48+rdi],rdx + sar rdx,63 + mov QWORD PTR[56+rdi],rdx + mov rdx,rcx + mov rax,rcx + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + mov r15,QWORD PTR[56+rsi] + mov rbx,QWORD PTR[64+rsi] + mov rbp,QWORD PTR[72+rsi] + mov rcx,QWORD PTR[80+rsi] + mov rdi,QWORD PTR[88+rsi] + + sar rax,63 + xor rsi,rsi + sub rsi,rax + + xor rdx,rax + add rdx,rsi + + xor r8,rax + xor r9,rax + xor r10,rax + xor r11,rax + xor r12,rax + xor r13,rax + xor r14,rax + xor r15,rax + xor rbx,rax + xor rbp,rax + xor rcx,rax + xor rdi,rax + add r8,rsi + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + adc r14,0 + adc r15,0 + adc rbx,0 + adc rbp,0 + adc rcx,0 + adc rdi,0 + + mulx rax,r8,r8 + mulx rsi,r9,r9 + add r9,rax + mulx rax,r10,r10 + adc r10,rsi + mulx rsi,r11,r11 + adc r11,rax + mulx rax,r12,r12 + adc r12,rsi + mulx rsi,r13,r13 + adc r13,rax + mulx rax,r14,r14 + adc r14,rsi + mulx rsi,r15,r15 + adc r15,rax + mulx rax,rbx,rbx + adc rbx,rsi + mulx rsi,rbp,rbp + adc rbp,rax + mulx rax,rcx,rcx + adc rcx,rsi + mulx rsi,rdi,rdi + mov rdx,QWORD PTR[8+rsp] + mov rsi,QWORD PTR[16+rsp] + adc rax,rdi + + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + adc r10,QWORD PTR[16+rdx] + adc r11,QWORD PTR[24+rdx] + adc r12,QWORD PTR[32+rdx] + adc r13,QWORD PTR[40+rdx] + adc r14,QWORD PTR[48+rdx] + mov rdi,QWORD PTR[56+rdx] + adc r15,rdi + adc rbx,rdi + adc rbp,rdi + adc rcx,rdi + adc rax,rdi + + mov rdi,rdx + + mov QWORD PTR[rdx],r8 + mov QWORD PTR[8+rdx],r9 + mov QWORD PTR[16+rdx],r10 + mov QWORD PTR[24+rdx],r11 + mov QWORD PTR[32+rdx],r12 + mov QWORD PTR[40+rdx],r13 + mov QWORD PTR[48+rdx],r14 + mov QWORD PTR[56+rdx],r15 + mov QWORD PTR[64+rdx],rbx + mov QWORD PTR[72+rdx],rbp + mov QWORD PTR[80+rdx],rcx + mov QWORD PTR[88+rdx],rax + + DB 0F3h,0C3h ;repret +__smulx_767x63 ENDP + +ALIGN 32 +__smulx_383x63 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[((0+0))+rsi] + mov r9,QWORD PTR[((0+8))+rsi] + mov r10,QWORD PTR[((0+16))+rsi] + mov r11,QWORD PTR[((0+24))+rsi] + mov r12,QWORD PTR[((0+32))+rsi] + mov r13,QWORD PTR[((0+40))+rsi] + + mov rbp,rdx + sar rbp,63 + xor rax,rax + sub rax,rbp + + xor rdx,rbp + add rdx,rax + + xor r8,rbp + xor r9,rbp + xor r10,rbp + xor r11,rbp + xor r12,rbp + xor r13,rbp + add r8,rax + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mulx rbp,r8,r8 + mulx rax,r9,r9 + add r9,rbp + mulx rbp,r10,r10 + adc r10,rax + mulx rax,r11,r11 + adc r11,rbp + mulx rbp,r12,r12 + adc r12,rax + mulx rax,r13,r13 + mov rdx,rcx + adc r13,rbp + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov r8,QWORD PTR[((48+0))+rsi] + mov r9,QWORD PTR[((48+8))+rsi] + mov r10,QWORD PTR[((48+16))+rsi] + mov r11,QWORD PTR[((48+24))+rsi] + mov r12,QWORD PTR[((48+32))+rsi] + mov r13,QWORD PTR[((48+40))+rsi] + + mov rbp,rdx + sar rbp,63 + xor rax,rax + sub rax,rbp + + xor rdx,rbp + add rdx,rax + + xor r8,rbp + xor r9,rbp + xor r10,rbp + xor r11,rbp + xor r12,rbp + xor r13,rbp + add r8,rax + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + mulx rbp,r8,r8 + mulx rax,r9,r9 + add r9,rbp + mulx rbp,r10,r10 + adc r10,rax + mulx rax,r11,r11 + adc r11,rbp + mulx rbp,r12,r12 + adc r12,rax + mulx rax,r13,r13 + adc r13,rbp + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__smulx_383x63 ENDP + +ALIGN 32 +__smulx_383_n_shift_by_31 PROC PRIVATE + DB 243,15,30,250 + mov rbx,rdx + xor r14,r14 + mov r8,QWORD PTR[((0+0))+rsi] + mov r9,QWORD PTR[((0+8))+rsi] + mov r10,QWORD PTR[((0+16))+rsi] + mov r11,QWORD PTR[((0+24))+rsi] + mov r12,QWORD PTR[((0+32))+rsi] + mov r13,QWORD PTR[((0+40))+rsi] + + mov rax,rdx + sar rax,63 + xor rbp,rbp + sub rbp,rax + + xor rdx,rax + add rdx,rbp + + xor r8,rax + xor r9,rax + xor r10,rax + xor r11,rax + xor r12,rax + xor rax,r13 + add r8,rbp + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc rax,0 + + mulx rbp,r8,r8 + mulx r13,r9,r9 + add r9,rbp + mulx rbp,r10,r10 + adc r10,r13 + mulx r13,r11,r11 + adc r11,rbp + mulx rbp,r12,r12 + adc r12,r13 + adc rbp,0 + imul rdx + add rax,rbp + adc r14,rdx + + mov rdx,rcx + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],rax + mov r8,QWORD PTR[((48+0))+rsi] + mov r9,QWORD PTR[((48+8))+rsi] + mov r10,QWORD PTR[((48+16))+rsi] + mov r11,QWORD PTR[((48+24))+rsi] + mov r12,QWORD PTR[((48+32))+rsi] + mov r13,QWORD PTR[((48+40))+rsi] + + mov rax,rdx + sar rax,63 + xor rbp,rbp + sub rbp,rax + + xor rdx,rax + add rdx,rbp + + xor r8,rax + xor r9,rax + xor r10,rax + xor r11,rax + xor r12,rax + xor rax,r13 + add r8,rbp + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc rax,0 + + mulx rbp,r8,r8 + mulx r13,r9,r9 + add r9,rbp + mulx rbp,r10,r10 + adc r10,r13 + mulx r13,r11,r11 + adc r11,rbp + mulx rbp,r12,r12 + adc r12,r13 + adc rbp,0 + imul rdx + add rax,rbp + adc rdx,0 + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc rax,QWORD PTR[40+rdi] + adc r14,rdx + mov rdx,rbx + + shrd r8,r9,31 + shrd r9,r10,31 + shrd r10,r11,31 + shrd r11,r12,31 + shrd r12,rax,31 + shrd rax,r14,31 + + sar r14,63 + xor rbp,rbp + sub rbp,r14 + + xor r8,r14 + xor r9,r14 + xor r10,r14 + xor r11,r14 + xor r12,r14 + xor rax,r14 + add r8,rbp + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc rax,0 + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],rax + + xor rdx,r14 + xor rcx,r14 + add rdx,rbp + add rcx,rbp + + DB 0F3h,0C3h ;repret +__smulx_383_n_shift_by_31 ENDP + +ALIGN 32 +__smulx_191_n_shift_by_31 PROC PRIVATE + DB 243,15,30,250 + mov rbx,rdx + mov r8,QWORD PTR[((0+0))+rsi] + mov r9,QWORD PTR[((0+8))+rsi] + mov r10,QWORD PTR[((0+16))+rsi] + + mov rax,rdx + sar rax,63 + xor rbp,rbp + sub rbp,rax + + xor rdx,rax + add rdx,rbp + + xor r8,rax + xor r9,rax + xor rax,r10 + add r8,rbp + adc r9,0 + adc rax,0 + + mulx rbp,r8,r8 + mulx r10,r9,r9 + add r9,rbp + adc r10,0 + imul rdx + add r10,rax + adc rdx,0 + mov r14,rdx + mov rdx,rcx + mov r11,QWORD PTR[((48+0))+rsi] + mov r12,QWORD PTR[((48+8))+rsi] + mov r13,QWORD PTR[((48+16))+rsi] + + mov rax,rdx + sar rax,63 + xor rbp,rbp + sub rbp,rax + + xor rdx,rax + add rdx,rbp + + xor r11,rax + xor r12,rax + xor rax,r13 + add r11,rbp + adc r12,0 + adc rax,0 + + mulx rbp,r11,r11 + mulx r13,r12,r12 + add r12,rbp + adc r13,0 + imul rdx + add r13,rax + adc rdx,0 + add r11,r8 + adc r12,r9 + adc r13,r10 + adc r14,rdx + mov rdx,rbx + + shrd r11,r12,31 + shrd r12,r13,31 + shrd r13,r14,31 + + sar r14,63 + xor rbp,rbp + sub rbp,r14 + + xor r11,r14 + xor r12,r14 + xor r13,r14 + add r11,rbp + adc r12,0 + adc r13,0 + + mov QWORD PTR[rdi],r11 + mov QWORD PTR[8+rdi],r12 + mov QWORD PTR[16+rdi],r13 + + xor rdx,r14 + xor rcx,r14 + add rdx,rbp + add rcx,rbp + + DB 0F3h,0C3h ;repret +__smulx_191_n_shift_by_31 ENDP + +ALIGN 32 +__ab_approximation_31 PROC PRIVATE + DB 243,15,30,250 + mov r9,QWORD PTR[40+rsi] + mov r11,QWORD PTR[88+rsi] + mov rbx,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[80+rsi] + mov r8,QWORD PTR[24+rsi] + mov r10,QWORD PTR[72+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + mov r8,QWORD PTR[16+rsi] + cmovz rbp,r10 + mov r10,QWORD PTR[64+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + mov r8,QWORD PTR[8+rsi] + cmovz rbp,r10 + mov r10,QWORD PTR[56+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + mov r8,QWORD PTR[rsi] + cmovz rbp,r10 + mov r10,QWORD PTR[48+rsi] + + mov rax,r9 + or rax,r11 + cmovz r9,rbx + cmovz r11,rbp + cmovz rbx,r8 + cmovz rbp,r10 + + mov rax,r9 + or rax,r11 + bsr rcx,rax + lea rcx,QWORD PTR[1+rcx] + cmovz r9,r8 + cmovz r11,r10 + cmovz rcx,rax + neg rcx + + + shld r9,rbx,cl + shld r11,rbp,cl + + mov eax,07FFFFFFFh + and r8,rax + and r10,rax + andn r9,rax,r9 + andn r11,rax,r11 + or r8,r9 + or r10,r11 + + jmp __inner_loop_31 + + DB 0F3h,0C3h ;repret +__ab_approximation_31 ENDP + +ALIGN 32 +__inner_loop_31 PROC PRIVATE + DB 243,15,30,250 + mov rcx,07FFFFFFF80000000h + mov r13,0800000007FFFFFFFh + mov r15,07FFFFFFF7FFFFFFFh + +$L$oop_31:: + cmp r8,r10 + mov rax,r8 + mov rbx,r10 + mov rbp,rcx + mov r14,r13 + cmovb r8,r10 + cmovb r10,rax + cmovb rcx,r13 + cmovb r13,rbp + + sub r8,r10 + sub rcx,r13 + add rcx,r15 + + test rax,1 + cmovz r8,rax + cmovz r10,rbx + cmovz rcx,rbp + cmovz r13,r14 + + shr r8,1 + add r13,r13 + sub r13,r15 + sub edi,1 + jnz $L$oop_31 + + shr r15,32 + mov edx,ecx + mov r12d,r13d + shr rcx,32 + shr r13,32 + sub rdx,r15 + sub rcx,r15 + sub r12,r15 + sub r13,r15 + + DB 0F3h,0C3h ;repret +__inner_loop_31 ENDP + + +ALIGN 32 +__inner_loop_62 PROC PRIVATE + DB 243,15,30,250 + mov rdx,1 + xor rcx,rcx + xor r12,r12 + mov r13,1 + +$L$oop_62:: + xor rax,rax + test r8,1 + mov rbx,r10 + cmovnz rax,r10 + sub rbx,r8 + mov rbp,r8 + sub r8,rax + cmovc r8,rbx + cmovc r10,rbp + mov rax,rdx + cmovc rdx,r12 + cmovc r12,rax + mov rbx,rcx + cmovc rcx,r13 + cmovc r13,rbx + xor rax,rax + xor rbx,rbx + shr r8,1 + test rbp,1 + cmovnz rax,r12 + cmovnz rbx,r13 + add r12,r12 + add r13,r13 + sub rdx,rax + sub rcx,rbx + sub edi,1 + jnz $L$oop_62 + + DB 0F3h,0C3h ;repret +__inner_loop_62 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_ctx_inverse_mod_383 + DD imagerel $L$SEH_body_ctx_inverse_mod_383 + DD imagerel $L$SEH_info_ctx_inverse_mod_383_prologue + + DD imagerel $L$SEH_body_ctx_inverse_mod_383 + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 + DD imagerel $L$SEH_info_ctx_inverse_mod_383_body + + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 + DD imagerel $L$SEH_end_ctx_inverse_mod_383 + DD imagerel $L$SEH_info_ctx_inverse_mod_383_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_ctx_inverse_mod_383_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_ctx_inverse_mod_383_body:: +DB 1,0,18,0 +DB 000h,0f4h,08bh,000h +DB 000h,0e4h,08ch,000h +DB 000h,0d4h,08dh,000h +DB 000h,0c4h,08eh,000h +DB 000h,034h,08fh,000h +DB 000h,054h,090h,000h +DB 000h,074h,092h,000h +DB 000h,064h,093h,000h +DB 000h,001h,091h,000h +$L$SEH_info_ctx_inverse_mod_383_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/div3w-armv8.asm b/crypto/blst_src/build/win64/div3w-armv8.asm new file mode 100644 index 00000000000..7114ccf0c2e --- /dev/null +++ b/crypto/blst_src/build/win64/div3w-armv8.asm @@ -0,0 +1,89 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + EXPORT |div_3_limbs|[FUNC] + ALIGN 32 +|div_3_limbs| PROC + ldp x4,x5,[x0] // load R + eor x0,x0,x0 // Q = 0 + mov x3,#64 // loop counter + nop + +|$Loop| + subs x6,x4,x1 // R - D + add x0,x0,x0 // Q <<= 1 + sbcs x7,x5,x2 + add x0,x0,#1 // Q + speculative bit + csello x4,x4,x6 + extr x1,x2,x1,#1 // D >>= 1 + csello x5,x5,x7 + lsr x2,x2,#1 + sbc x0,x0,xzr // subtract speculative bit + sub x3,x3,#1 + cbnz x3,|$Loop| + + asr x3,x0,#63 // top bit -> mask + add x0,x0,x0 // Q <<= 1 + subs x6,x4,x1 // R - D + add x0,x0,#1 // Q + specilative bit + sbcs x7,x5,x2 + sbc x0,x0,xzr // subtract speculative bit + + orr x0,x0,x3 // all ones if overflow + + ret + ENDP + + EXPORT |quot_rem_128|[FUNC] + ALIGN 32 +|quot_rem_128| PROC + ldp x3,x4,[x1] + + mul x5,x3,x2 // divisor[0:1} * quotient + umulh x6,x3,x2 + mul x11, x4,x2 + umulh x7,x4,x2 + + ldp x8,x9,[x0] // load 3 limbs of the dividend + ldr x10,[x0,#16] + + adds x6,x6,x11 + adc x7,x7,xzr + + subs x8,x8,x5 // dividend - divisor * quotient + sbcs x9,x9,x6 + sbcs x10,x10,x7 + sbc x5,xzr,xzr // borrow -> mask + + add x2,x2,x5 // if borrowed, adjust the quotient ... + and x3,x3,x5 + and x4,x4,x5 + adds x8,x8,x3 // ... and add divisor + adc x9,x9,x4 + + stp x8,x9,[x0] // save 2 limbs of the remainder + str x2,[x0,#16] // and one limb of the quotient + + mov x0,x2 // return adjusted quotient + + ret + ENDP + + + EXPORT |quot_rem_64|[FUNC] + ALIGN 32 +|quot_rem_64| PROC + ldr x3,[x1] + ldr x8,[x0] // load 1 limb of the dividend + + mul x5,x3,x2 // divisor * quotient + + sub x8,x8,x5 // dividend - divisor * quotient + + stp x8,x2,[x0] // save remainder and quotient + + mov x0,x2 // return quotient + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/div3w-x86_64.asm b/crypto/blst_src/build/win64/div3w-x86_64.asm new file mode 100644 index 00000000000..c35f426f3d2 --- /dev/null +++ b/crypto/blst_src/build/win64/div3w-x86_64.asm @@ -0,0 +1,152 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC div_3_limbs + + +ALIGN 32 +div_3_limbs PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_div_3_limbs:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + xor rax,rax + mov ecx,64 + +$L$oop:: + mov r10,r8 + sub r8,rsi + mov r11,r9 + sbb r9,rdx + lea rax,QWORD PTR[1+rax*1+rax] + mov rdi,rdx + cmovc r8,r10 + cmovc r9,r11 + sbb rax,0 + shl rdi,63 + shr rsi,1 + shr rdx,1 + or rsi,rdi + sub ecx,1 + jnz $L$oop + + lea rcx,QWORD PTR[1+rax*1+rax] + sar rax,63 + + sub r8,rsi + sbb r9,rdx + sbb rcx,0 + + or rax,rcx + + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_div_3_limbs:: +div_3_limbs ENDP +PUBLIC quot_rem_128 + + +ALIGN 32 +quot_rem_128 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_quot_rem_128:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + mov rax,rdx + mov rcx,rdx + + mul QWORD PTR[rsi] + mov r8,rax + mov rax,rcx + mov r9,rdx + + mul QWORD PTR[8+rsi] + add r9,rax + adc rdx,0 + + mov r10,QWORD PTR[rdi] + mov r11,QWORD PTR[8+rdi] + mov rax,QWORD PTR[16+rdi] + + sub r10,r8 + sbb r11,r9 + sbb rax,rdx + sbb r8,r8 + + add rcx,r8 + mov r9,r8 + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + add r10,r8 + adc r11,r9 + + mov QWORD PTR[rdi],r10 + mov QWORD PTR[8+rdi],r11 + mov QWORD PTR[16+rdi],rcx + + mov rax,rcx + + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_quot_rem_128:: +quot_rem_128 ENDP + + + + + +PUBLIC quot_rem_64 + + +ALIGN 32 +quot_rem_64 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_quot_rem_64:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + mov rax,rdx + imul rdx,QWORD PTR[rsi] + + mov r10,QWORD PTR[rdi] + + sub r10,rdx + + mov QWORD PTR[rdi],r10 + mov QWORD PTR[8+rdi],rax + + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + DB 0F3h,0C3h ;repret +$L$SEH_end_quot_rem_64:: +quot_rem_64 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/dll.c b/crypto/blst_src/build/win64/dll.c new file mode 100644 index 00000000000..a70d0c98a23 --- /dev/null +++ b/crypto/blst_src/build/win64/dll.c @@ -0,0 +1,32 @@ +#include <windows.h> + +#if defined(_MSC_VER) +/* + * Even though we don't have memcpy/memset anywhere, MSVC compiler + * generates calls to them as it recognizes corresponding patterns. + */ +void *memcpy(unsigned char *dst, const unsigned char *src, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = *src++; + + return ret; +} + +void *memset(unsigned char *dst, int c, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = (unsigned char)c; + + return ret; +} +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) +{ return TRUE; } diff --git a/crypto/blst_src/build/win64/mul_mont_256-armv8.asm b/crypto/blst_src/build/win64/mul_mont_256-armv8.asm new file mode 100644 index 00000000000..bb2dfe043c7 --- /dev/null +++ b/crypto/blst_src/build/win64/mul_mont_256-armv8.asm @@ -0,0 +1,465 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + + EXPORT |mul_mont_sparse_256|[FUNC] + ALIGN 32 +|mul_mont_sparse_256| PROC + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x10,x11,[x1] + ldr x9, [x2] + ldp x12,x13,[x1,#16] + + mul x19,x10,x9 + ldp x5,x6,[x3] + mul x20,x11,x9 + ldp x7,x8,[x3,#16] + mul x21,x12,x9 + mul x22,x13,x9 + + umulh x14,x10,x9 + umulh x15,x11,x9 + mul x3,x4,x19 + umulh x16,x12,x9 + umulh x17,x13,x9 + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,xzr, x17 + mul x17,x8,x3 + ldr x9,[x2,8*1] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*2] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + ldr x9,[x2,8*3] + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + mul x14,x10,x9 + adcs x20,x21,x15 + mul x15,x11,x9 + adcs x21,x22,x16 + mul x16,x12,x9 + adcs x22,x23,x17 + mul x17,x13,x9 + adc x23,xzr,xzr + + adds x19,x19,x14 + umulh x14,x10,x9 + adcs x20,x20,x15 + umulh x15,x11,x9 + adcs x21,x21,x16 + mul x3,x4,x19 + umulh x16,x12,x9 + adcs x22,x22,x17 + umulh x17,x13,x9 + adc x23,x23,xzr + + adds x20,x20,x14 + //mul x14,x5,x3 + adcs x21,x21,x15 + mul x15,x6,x3 + adcs x22,x22,x16 + mul x16,x7,x3 + adc x23,x23,x17 + mul x17,x8,x3 + subs xzr,x19,#1 //adds x19,x19,x14 + umulh x14,x5,x3 + adcs x20,x20,x15 + umulh x15,x6,x3 + adcs x21,x21,x16 + umulh x16,x7,x3 + adcs x22,x22,x17 + umulh x17,x8,x3 + adc x23,x23,xzr + + adds x19,x20,x14 + adcs x20,x21,x15 + adcs x21,x22,x16 + adcs x22,x23,x17 + adc x23,xzr,xzr + + subs x14,x19,x5 + sbcs x15,x20,x6 + sbcs x16,x21,x7 + sbcs x17,x22,x8 + sbcs xzr, x23,xzr + + csello x19,x19,x14 + csello x20,x20,x15 + csello x21,x21,x16 + csello x22,x22,x17 + + stp x19,x20,[x0] + stp x21,x22,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + ret + ENDP + + + EXPORT |sqr_mont_sparse_256|[FUNC] + ALIGN 32 +|sqr_mont_sparse_256| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-48]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + mov x4,x3 + + //////////////////////////////////////////////////////////////// + // | | | | | |a1*a0| | + // | | | | |a2*a0| | | + // | |a3*a2|a3*a0| | | | + // | | | |a2*a1| | | | + // | | |a3*a1| | | | | + // *| | | | | | | | 2| + // +|a3*a3|a2*a2|a1*a1|a0*a0| + // |--+--+--+--+--+--+--+--| + // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is x10 + // + // "can't overflow" below mark carrying into high part of + // multiplication result, which can't overflow, because it + // can never be all ones. + + mul x11,x6,x5 // a[1]*a[0] + umulh x15,x6,x5 + mul x12,x7,x5 // a[2]*a[0] + umulh x16,x7,x5 + mul x13,x8,x5 // a[3]*a[0] + umulh x19,x8,x5 + + adds x12,x12,x15 // accumulate high parts of multiplication + mul x14,x7,x6 // a[2]*a[1] + umulh x15,x7,x6 + adcs x13,x13,x16 + mul x16,x8,x6 // a[3]*a[1] + umulh x17,x8,x6 + adc x19,x19,xzr // can't overflow + + mul x20,x8,x7 // a[3]*a[2] + umulh x21,x8,x7 + + adds x15,x15,x16 // accumulate high parts of multiplication + mul x10,x5,x5 // a[0]*a[0] + adc x16,x17,xzr // can't overflow + + adds x13,x13,x14 // accumulate low parts of multiplication + umulh x5,x5,x5 + adcs x19,x19,x15 + mul x15,x6,x6 // a[1]*a[1] + adcs x20,x20,x16 + umulh x6,x6,x6 + adc x21,x21,xzr // can't overflow + + adds x11,x11,x11 // acc[1-6]*=2 + mul x16,x7,x7 // a[2]*a[2] + adcs x12,x12,x12 + umulh x7,x7,x7 + adcs x13,x13,x13 + mul x17,x8,x8 // a[3]*a[3] + adcs x19,x19,x19 + umulh x8,x8,x8 + adcs x20,x20,x20 + adcs x21,x21,x21 + adc x22,xzr,xzr + + adds x11,x11,x5 // +a[i]*a[i] + adcs x12,x12,x15 + adcs x13,x13,x6 + adcs x19,x19,x16 + adcs x20,x20,x7 + adcs x21,x21,x17 + adc x22,x22,x8 + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + adds x10,x10,x19 // accumulate upper half + adcs x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adc x19,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x19,xzr + + csello x10,x10,x14 + csello x11,x11,x15 + csello x12,x12,x16 + csello x13,x13,x17 + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldr x29,[sp],#48 + DCDU 3573752767 + ret + ENDP + + + EXPORT |from_mont_256|[FUNC] + ALIGN 32 +|from_mont_256| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + + csello x10,x10,x14 + csello x11,x11,x15 + csello x12,x12,x16 + csello x13,x13,x17 + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |redc_mont_256|[FUNC] + ALIGN 32 +|redc_mont_256| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x4,x3 + ldp x10,x11,[x1] + ldp x12,x13,[x1,#16] + + bl __mul_by_1_mont_256 + ldr x30,[x29,#8] + + ldp x14,x15,[x1,#32] + ldp x16,x17,[x1,#48] + + adds x10,x10,x14 + adcs x11,x11,x15 + adcs x12,x12,x16 + adcs x13,x13,x17 + adc x9,xzr,xzr + + subs x14,x10,x5 + sbcs x15,x11,x6 + sbcs x16,x12,x7 + sbcs x17,x13,x8 + sbcs xzr, x9,xzr + + csello x10,x10,x14 + csello x11,x11,x15 + csello x12,x12,x16 + csello x13,x13,x17 + + stp x10,x11,[x0] + stp x12,x13,[x0,#16] + + ldr x29,[sp],#16 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__mul_by_1_mont_256| PROC + mul x3,x4,x10 + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + mul x3,x4,x10 + adc x13,x9,x17 + //mul x14,x5,x3 + mul x15,x6,x3 + mul x16,x7,x3 + mul x17,x8,x3 + subs xzr,x10,#1 //adds x10,x10,x14 + umulh x14,x5,x3 + adcs x11,x11,x15 + umulh x15,x6,x3 + adcs x12,x12,x16 + umulh x16,x7,x3 + adcs x13,x13,x17 + umulh x17,x8,x3 + adc x9,xzr,xzr + + adds x10,x11,x14 + adcs x11,x12,x15 + adcs x12,x13,x16 + adc x13,x9,x17 + + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/mul_mont_384-armv8.asm b/crypto/blst_src/build/win64/mul_mont_384-armv8.asm new file mode 100644 index 00000000000..a309dfa4121 --- /dev/null +++ b/crypto/blst_src/build/win64/mul_mont_384-armv8.asm @@ -0,0 +1,2373 @@ + AREA |.text|,CODE,ALIGN=8,ARM64 + + + EXPORT |add_mod_384x384|[FUNC] + ALIGN 32 +|add_mod_384x384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __add_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__add_mod_384x384| PROC + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + stp x11, x12, [x0] + adcs x15,x15,x23 + ldp x11, x12, [x1,#48] + adcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + adcs x11,x11,x19 + stp x15, x16, [x0,#32] + adcs x12,x12,x20 + ldp x15, x16, [x1,#80] + adcs x13,x13,x21 + ldp x23,x24,[x2,#80] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csello x11,x11,x19 + csello x12,x12,x20 + csello x13,x13,x21 + csello x14,x14,x22 + stp x11,x12,[x0,#48] + csello x15,x15,x23 + stp x13,x14,[x0,#64] + csello x16,x16,x24 + stp x15,x16,[x0,#80] + + ret + ENDP + + + EXPORT |sub_mod_384x384|[FUNC] + ALIGN 32 +|sub_mod_384x384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-64]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldr x29,[sp],#64 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__sub_mod_384x384| PROC + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + stp x11, x12, [x0] + sbcs x15,x15,x23 + ldp x11, x12, [x1,#48] + sbcs x16,x16,x24 + + ldp x19,x20,[x2,#48] + stp x13, x14, [x0,#16] + ldp x13, x14, [x1,#64] + ldp x21,x22,[x2,#64] + + sbcs x11,x11,x19 + stp x15, x16, [x0,#32] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#80] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#80] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + ret + ENDP + + + ALIGN 32 +|__add_mod_384| PROC + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + adds x11,x11,x19 + ldp x21,x22,[x2,#16] + adcs x12,x12,x20 + ldp x15, x16, [x1,#32] + adcs x13,x13,x21 + ldp x23,x24,[x2,#32] + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x17,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x17,xzr + + csello x11,x11,x19 + csello x12,x12,x20 + csello x13,x13,x21 + csello x14,x14,x22 + csello x15,x15,x23 + stp x11,x12,[x0] + csello x16,x16,x24 + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + ENDP + + + ALIGN 32 +|__sub_mod_384| PROC + ldp x11, x12, [x1] + ldp x19,x20,[x2] + ldp x13, x14, [x1,#16] + subs x11,x11,x19 + ldp x21,x22,[x2,#16] + sbcs x12,x12,x20 + ldp x15, x16, [x1,#32] + sbcs x13,x13,x21 + ldp x23,x24,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x17,xzr,xzr + + and x19,x5,x17 + and x20,x6,x17 + adds x11,x11,x19 + and x21,x7,x17 + adcs x12,x12,x20 + and x22,x8,x17 + adcs x13,x13,x21 + and x23,x9,x17 + adcs x14,x14,x22 + and x24,x10,x17 + adcs x15,x15,x23 + stp x11,x12,[x0] + adc x16,x16,x24 + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + ENDP + + + + EXPORT |mul_mont_384x|[FUNC] + ALIGN 32 +|mul_mont_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#288 // space for 3 768-bit vectors + + mov x26,x0 // save r_ptr + mov x27,x1 // save b_ptr + mov x28,x2 // save b_ptr + + sub x0,sp,#0 // mul_384(t0, a->re, b->re) + bl __mul_384 + + add x1,x1,#48 // mul_384(t1, a->im, b->im) + add x2,x2,#48 + add x0,sp,#96 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + sub x2,x1,#48 + add x0,sp,#240 + bl __add_mod_384 + + add x1,x28,#0 + add x2,x28,#48 + add x0,sp,#192 // t2 + bl __add_mod_384 + + add x1,x0,#0 + add x2,x0,#48 + bl __mul_384 // mul_384(t2, a->re+a->im, b->re+b->im) + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,x0 + add x2,sp,#0 + bl __sub_mod_384x384 + + add x2,sp,#96 + bl __sub_mod_384x384 // t2 = t2-t0-t1 + + add x1,sp,#0 + add x2,sp,#96 + add x0,sp,#0 + bl __sub_mod_384x384 // t0 = t0-t1 + + add x1,sp,#0 // ret->re = redc(t0) + add x0,x26,#0 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + + add x1,sp,#192 // ret->im = redc(t2) + add x0,x0,#48 + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#288 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sqr_mont_384x|[FUNC] + ALIGN 32 +|sqr_mont_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 2 384-bit vectors + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + add x0,sp,#0 + bl __add_mod_384 // t0 = a->re + a->im + + add x0,sp,#48 + bl __sub_mod_384 // t1 = a->re - a->im + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __mul_mont_384 // mul_mont_384(ret->im, a->re, a->im) + + adds x11,x11,x11 // add with itself + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csello x19,x11,x19 + csello x20,x12,x20 + csello x21,x13,x21 + ldp x11,x12,[sp] + csello x22,x14,x22 + ldr x17, [sp,#48] + csello x23,x15,x23 + ldp x13,x14,[sp,#16] + csello x24,x16,x24 + ldp x15,x16,[sp,#32] + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + add x2,sp,#48 + bl __mul_mont_384 // mul_mont_384(ret->re, t0, t1) + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |mul_mont_384|[FUNC] + ALIGN 32 +|mul_mont_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__mul_mont_384| PROC + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + mov x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*1] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*2] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*3] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*4] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + adc x4,x17,xzr + ldr x17,[x2,8*5] + + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,x4,xzr + ldr x4,[x29,#96] + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adcs x25,x25,xzr + adc x17,xzr,xzr + + adds x20,x20,x26 + // mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adcs x25,x25,x3 + mul x3,x10,x4 + adc x17,x17,xzr + subs xzr,x19,#1 // adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adcs x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + adc x17,x17,xzr + + adds x19,x20,x26 + adcs x20,x21,x27 + adcs x21,x22,x28 + adcs x22,x23,x0 + adcs x23,x24,x1 + adcs x24,x25,x3 + adc x25,x17,xzr + + subs x26,x19,x5 + sbcs x27,x20,x6 + sbcs x28,x21,x7 + sbcs x0,x22,x8 + sbcs x1,x23,x9 + sbcs x3,x24,x10 + sbcs xzr, x25,xzr + + csello x11,x19,x26 + csello x12,x20,x27 + csello x13,x21,x28 + csello x14,x22,x0 + csello x15,x23,x1 + csello x16,x24,x3 + ret + ENDP + + + + EXPORT |sqr_mont_384|[FUNC] + ALIGN 32 +|sqr_mont_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for 768-bit vector + mov x4,x3 // adjust for missing b_ptr + + mov x3,x0 // save r_ptr + mov x0,sp + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + mov x1,sp + mov x0,x3 // restore r_ptr + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sqr_n_mul_mont_383|[FUNC] + ALIGN 32 +|sqr_n_mul_mont_383| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x4,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#96 // space for 768-bit vector + mov x17,x5 // save b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + mov x0,sp +|$Loop_sqr_383| + bl __sqr_384 + sub x2,x2,#1 // counter + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + mov x1,sp + bl __mul_by_1_mont_384 + + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // just accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + cbnz x2,|$Loop_sqr_383| + + mov x2,x17 + ldr x17,[x17] + bl __mul_mont_384 + ldr x30,[x29,#8] + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + ALIGN 32 +|__sqr_384| PROC + mul x19,x12,x11 + mul x20,x13,x11 + mul x21,x14,x11 + mul x22,x15,x11 + mul x23,x16,x11 + + umulh x6,x12,x11 + umulh x7,x13,x11 + umulh x8,x14,x11 + umulh x9,x15,x11 + adds x20,x20,x6 + umulh x10,x16,x11 + adcs x21,x21,x7 + mul x7,x13,x12 + adcs x22,x22,x8 + mul x8,x14,x12 + adcs x23,x23,x9 + mul x9,x15,x12 + adc x24,xzr, x10 + mul x10,x16,x12 + + adds x21,x21,x7 + umulh x7,x13,x12 + adcs x22,x22,x8 + umulh x8,x14,x12 + adcs x23,x23,x9 + umulh x9,x15,x12 + adcs x24,x24,x10 + umulh x10,x16,x12 + adc x25,xzr,xzr + + mul x5,x11,x11 + adds x22,x22,x7 + umulh x11, x11,x11 + adcs x23,x23,x8 + mul x8,x14,x13 + adcs x24,x24,x9 + mul x9,x15,x13 + adc x25,x25,x10 + mul x10,x16,x13 + + adds x23,x23,x8 + umulh x8,x14,x13 + adcs x24,x24,x9 + umulh x9,x15,x13 + adcs x25,x25,x10 + umulh x10,x16,x13 + adc x26,xzr,xzr + + mul x6,x12,x12 + adds x24,x24,x8 + umulh x12, x12,x12 + adcs x25,x25,x9 + mul x9,x15,x14 + adc x26,x26,x10 + mul x10,x16,x14 + + adds x25,x25,x9 + umulh x9,x15,x14 + adcs x26,x26,x10 + umulh x10,x16,x14 + adc x27,xzr,xzr + mul x7,x13,x13 + adds x26,x26,x9 + umulh x13, x13,x13 + adc x27,x27,x10 + mul x8,x14,x14 + + mul x10,x16,x15 + umulh x14, x14,x14 + adds x27,x27,x10 + umulh x10,x16,x15 + mul x9,x15,x15 + adc x28,x10,xzr + + adds x19,x19,x19 + adcs x20,x20,x20 + adcs x21,x21,x21 + adcs x22,x22,x22 + adcs x23,x23,x23 + adcs x24,x24,x24 + adcs x25,x25,x25 + adcs x26,x26,x26 + umulh x15, x15,x15 + adcs x27,x27,x27 + mul x10,x16,x16 + adcs x28,x28,x28 + umulh x16, x16,x16 + adc x1,xzr,xzr + + adds x19,x19,x11 + adcs x20,x20,x6 + adcs x21,x21,x12 + adcs x22,x22,x7 + adcs x23,x23,x13 + adcs x24,x24,x8 + adcs x25,x25,x14 + stp x5,x19,[x0] + adcs x26,x26,x9 + stp x20,x21,[x0,#16] + adcs x27,x27,x15 + stp x22,x23,[x0,#32] + adcs x28,x28,x10 + stp x24,x25,[x0,#48] + adc x16,x16,x1 + stp x26,x27,[x0,#64] + stp x28,x16,[x0,#80] + + ret + ENDP + + + EXPORT |sqr_384|[FUNC] + ALIGN 32 +|sqr_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + bl __sqr_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |redc_mont_384|[FUNC] + ALIGN 32 +|redc_mont_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + bl __redc_tail_mont_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |from_mont_384|[FUNC] + ALIGN 32 +|from_mont_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + mov x4,x3 // adjust for missing b_ptr + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + + csello x11,x11,x19 + csello x12,x12,x20 + csello x13,x13,x21 + csello x14,x14,x22 + csello x15,x15,x23 + csello x16,x16,x24 + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__mul_by_1_mont_384| PROC + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + mul x26,x4,x11 + ldp x15,x16,[x1,#32] + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + mul x26,x4,x11 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + // mul x19,x5,x26 + mul x20,x6,x26 + mul x21,x7,x26 + mul x22,x8,x26 + mul x23,x9,x26 + mul x24,x10,x26 + subs xzr,x11,#1 // adds x19,x19,x11 + umulh x11,x5,x26 + adcs x20,x20,x12 + umulh x12,x6,x26 + adcs x21,x21,x13 + umulh x13,x7,x26 + adcs x22,x22,x14 + umulh x14,x8,x26 + adcs x23,x23,x15 + umulh x15,x9,x26 + adcs x24,x24,x16 + umulh x16,x10,x26 + adc x25,xzr,xzr + adds x11,x11,x20 + adcs x12,x12,x21 + adcs x13,x13,x22 + adcs x14,x14,x23 + adcs x15,x15,x24 + adc x16,x16,x25 + + ret + ENDP + + + ALIGN 32 +|__redc_tail_mont_384| PROC + ldp x19,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x11,x11,x19 // accumulate upper half + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adcs x16,x16,x24 + adc x25,xzr,xzr + + subs x19,x11,x5 + sbcs x20,x12,x6 + sbcs x21,x13,x7 + sbcs x22,x14,x8 + sbcs x23,x15,x9 + sbcs x24,x16,x10 + sbcs xzr,x25,xzr + + csello x11,x11,x19 + csello x12,x12,x20 + csello x13,x13,x21 + csello x14,x14,x22 + csello x15,x15,x23 + csello x16,x16,x24 + + stp x11,x12,[x0] + stp x13,x14,[x0,#16] + stp x15,x16,[x0,#32] + + ret + ENDP + + + + EXPORT |mul_384|[FUNC] + ALIGN 32 +|mul_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + bl __mul_384 + ldr x30,[x29,#8] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__mul_384| PROC + ldp x11,x12,[x1] + ldr x17, [x2] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + + umulh x5,x11,x17 + umulh x6,x12,x17 + umulh x7,x13,x17 + umulh x8,x14,x17 + umulh x9,x15,x17 + umulh x10,x16,x17 + ldr x17,[x2,8*1] + + str x19,[x0] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,xzr, x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(1+1)] + adc x25,xzr,xzr + + str x19,[x0,8*1] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(2+1)] + adc x25,xzr,xzr + + str x19,[x0,8*2] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(3+1)] + adc x25,xzr,xzr + + str x19,[x0,8*3] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + ldr x17,[x2,#8*(4+1)] + adc x25,xzr,xzr + + str x19,[x0,8*4] + adds x19,x20,x5 + mul x5,x11,x17 + adcs x20,x21,x6 + mul x6,x12,x17 + adcs x21,x22,x7 + mul x7,x13,x17 + adcs x22,x23,x8 + mul x8,x14,x17 + adcs x23,x24,x9 + mul x9,x15,x17 + adc x24,x25,x10 + mul x10,x16,x17 + adds x19,x19,x5 + umulh x5,x11,x17 + adcs x20,x20,x6 + umulh x6,x12,x17 + adcs x21,x21,x7 + umulh x7,x13,x17 + adcs x22,x22,x8 + umulh x8,x14,x17 + adcs x23,x23,x9 + umulh x9,x15,x17 + adcs x24,x24,x10 + umulh x10,x16,x17 + adc x25,xzr,xzr + + str x19,[x0,8*5] + adds x19,x20,x5 + adcs x20,x21,x6 + adcs x21,x22,x7 + adcs x22,x23,x8 + adcs x23,x24,x9 + adc x24,x25,x10 + + stp x19,x20,[x0,#48] + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ret + ENDP + + + + EXPORT |mul_382x|[FUNC] + ALIGN 32 +|mul_382x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#96 // space for two 384-bit vectors + + ldp x11,x12,[x1] + mov x26,x0 // save r_ptr + ldp x19,x20,[x1,#48] + mov x27,x1 // save a_ptr + ldp x13,x14,[x1,#16] + mov x28,x2 // save b_ptr + ldp x21,x22,[x1,#64] + ldp x15,x16,[x1,#32] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x23,x24,[x1,#80] + adcs x6,x12,x20 + ldp x11,x12,[x2] + adcs x7,x13,x21 + ldp x19,x20,[x2,#48] + adcs x8,x14,x22 + ldp x13,x14,[x2,#16] + adcs x9,x15,x23 + ldp x21,x22,[x2,#64] + adc x10,x16,x24 + ldp x15,x16,[x2,#32] + + stp x5,x6,[sp] + adds x5,x11,x19 // t1 = b->re + b->im + ldp x23,x24,[x2,#80] + adcs x6,x12,x20 + stp x7,x8,[sp,#16] + adcs x7,x13,x21 + adcs x8,x14,x22 + stp x9,x10,[sp,#32] + adcs x9,x15,x23 + stp x5,x6,[sp,#48] + adc x10,x16,x24 + stp x7,x8,[sp,#64] + stp x9,x10,[sp,#80] + + bl __mul_384 // mul_384(ret->re, a->re, b->re) + + add x1,sp,#0 // mul_384(ret->im, t0, t1) + add x2,sp,#48 + add x0,x26,#96 + bl __mul_384 + + add x1,x27,#48 // mul_384(tx, a->im, b->im) + add x2,x28,#48 + add x0,sp,#0 + bl __mul_384 + + ldp x5,x6,[x3] + ldp x7,x8,[x3,#16] + ldp x9,x10,[x3,#32] + + add x1,x26,#96 // ret->im -= tx + add x2,sp,#0 + add x0,x26,#96 + bl __sub_mod_384x384 + + add x2,x26,#0 // ret->im -= ret->re + bl __sub_mod_384x384 + + add x1,x26,#0 // ret->re -= tx + add x2,sp,#0 + add x0,x26,#0 + bl __sub_mod_384x384 + ldr x30,[x29,#8] + + add sp,sp,#96 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sqr_382x|[FUNC] + ALIGN 32 +|sqr_382x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + ldp x11,x12,[x1] + ldp x19,x20,[x1,#48] + ldp x13,x14,[x1,#16] + adds x5,x11,x19 // t0 = a->re + a->im + ldp x21,x22,[x1,#64] + adcs x6,x12,x20 + ldp x15,x16,[x1,#32] + adcs x7,x13,x21 + ldp x23,x24,[x1,#80] + adcs x8,x14,x22 + stp x5,x6,[x0] + adcs x9,x15,x23 + ldp x5,x6,[x2] + adc x10,x16,x24 + stp x7,x8,[x0,#16] + + subs x11,x11,x19 // t1 = a->re - a->im + ldp x7,x8,[x2,#16] + sbcs x12,x12,x20 + stp x9,x10,[x0,#32] + sbcs x13,x13,x21 + ldp x9,x10,[x2,#32] + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + adds x11,x11,x19 + and x21,x7,x25 + adcs x12,x12,x20 + and x22,x8,x25 + adcs x13,x13,x21 + and x23,x9,x25 + adcs x14,x14,x22 + and x24,x10,x25 + adcs x15,x15,x23 + stp x11,x12,[x0,#48] + adc x16,x16,x24 + stp x13,x14,[x0,#64] + stp x15,x16,[x0,#80] + + mov x4,x1 // save a_ptr + add x1,x0,#0 // mul_384(ret->re, t0, t1) + add x2,x0,#48 + bl __mul_384 + + add x1,x4,#0 // mul_384(ret->im, a->re, a->im) + add x2,x4,#48 + add x0,x0,#96 + bl __mul_384 + ldr x30,[x29,#8] + + ldp x11,x12,[x0] + ldp x13,x14,[x0,#16] + adds x11,x11,x11 // add with itself + ldp x15,x16,[x0,#32] + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adcs x19,x19,x19 + adcs x20,x20,x20 + stp x11,x12,[x0] + adcs x21,x21,x21 + stp x13,x14,[x0,#16] + adcs x22,x22,x22 + stp x15,x16,[x0,#32] + adcs x23,x23,x23 + stp x19,x20,[x0,#48] + adc x24,x24,x24 + stp x21,x22,[x0,#64] + stp x23,x24,[x0,#80] + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sqr_mont_382x|[FUNC] + ALIGN 32 +|sqr_mont_382x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + stp x3,x0,[sp,#96] // __mul_mont_384 wants them there + sub sp,sp,#112 // space for two 384-bit vectors + word + mov x4,x3 // adjust for missing b_ptr + + ldp x11,x12,[x1] + ldp x13,x14,[x1,#16] + ldp x15,x16,[x1,#32] + + ldp x17,x20,[x1,#48] + ldp x21,x22,[x1,#64] + ldp x23,x24,[x1,#80] + + adds x5,x11,x17 // t0 = a->re + a->im + adcs x6,x12,x20 + adcs x7,x13,x21 + adcs x8,x14,x22 + adcs x9,x15,x23 + adc x10,x16,x24 + + subs x19,x11,x17 // t1 = a->re - a->im + sbcs x20,x12,x20 + sbcs x21,x13,x21 + sbcs x22,x14,x22 + sbcs x23,x15,x23 + sbcs x24,x16,x24 + sbc x25,xzr,xzr // borrow flag as mask + + stp x5,x6,[sp] + stp x7,x8,[sp,#16] + stp x9,x10,[sp,#32] + stp x19,x20,[sp,#48] + stp x21,x22,[sp,#64] + stp x23,x24,[sp,#80] + str x25,[sp,#96] + + ldp x5,x6,[x2] + ldp x7,x8,[x2,#16] + ldp x9,x10,[x2,#32] + + add x2,x1,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) + + adds x19,x11,x11 // add with itself + adcs x20,x12,x12 + adcs x21,x13,x13 + adcs x22,x14,x14 + adcs x23,x15,x15 + adc x24,x16,x16 + + stp x19,x20,[x2,#48] + stp x21,x22,[x2,#64] + stp x23,x24,[x2,#80] + + ldp x11,x12,[sp] + ldr x17,[sp,#48] + ldp x13,x14,[sp,#16] + ldp x15,x16,[sp,#32] + + add x2,sp,#48 + bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) + ldr x30,[x29,#8] + + ldr x25,[sp,#96] // account for sign from a->re - a->im + ldp x19,x20,[sp] + ldp x21,x22,[sp,#16] + ldp x23,x24,[sp,#32] + + and x19,x19,x25 + and x20,x20,x25 + and x21,x21,x25 + and x22,x22,x25 + and x23,x23,x25 + and x24,x24,x25 + + subs x11,x11,x19 + sbcs x12,x12,x20 + sbcs x13,x13,x21 + sbcs x14,x14,x22 + sbcs x15,x15,x23 + sbcs x16,x16,x24 + sbc x25,xzr,xzr + + and x19,x5,x25 + and x20,x6,x25 + and x21,x7,x25 + and x22,x8,x25 + and x23,x9,x25 + and x24,x10,x25 + + adds x11,x11,x19 + adcs x12,x12,x20 + adcs x13,x13,x21 + adcs x14,x14,x22 + adcs x15,x15,x23 + adc x16,x16,x24 + + stp x11,x12,[x2] + stp x13,x14,[x2,#16] + stp x15,x16,[x2,#32] + + add sp,sp,#112 + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + ALIGN 32 +|__mul_mont_383_nonred| PROC + mul x19,x11,x17 + mul x20,x12,x17 + mul x21,x13,x17 + mul x22,x14,x17 + mul x23,x15,x17 + mul x24,x16,x17 + mul x4,x4,x19 + + umulh x26,x11,x17 + umulh x27,x12,x17 + umulh x28,x13,x17 + umulh x0,x14,x17 + umulh x1,x15,x17 + umulh x3,x16,x17 + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,xzr, x3 + mul x3,x10,x4 + ldr x17,[x2,8*1] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*2] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*3] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*4] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + ldr x17,[x2,8*5] + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + + ldr x4,[x29,#96] + adds x19,x20,x26 + mul x26,x11,x17 + adcs x20,x21,x27 + mul x27,x12,x17 + adcs x21,x22,x28 + mul x28,x13,x17 + adcs x22,x23,x0 + mul x0,x14,x17 + adcs x23,x24,x1 + mul x1,x15,x17 + adcs x24,x25,x3 + mul x3,x16,x17 + adc x25,xzr,xzr + + adds x19,x19,x26 + umulh x26,x11,x17 + adcs x20,x20,x27 + umulh x27,x12,x17 + adcs x21,x21,x28 + mul x4,x4,x19 + umulh x28,x13,x17 + adcs x22,x22,x0 + umulh x0,x14,x17 + adcs x23,x23,x1 + umulh x1,x15,x17 + adcs x24,x24,x3 + umulh x3,x16,x17 + adc x25,x25,xzr + + adds x20,x20,x26 + mul x26,x5,x4 + adcs x21,x21,x27 + mul x27,x6,x4 + adcs x22,x22,x28 + mul x28,x7,x4 + adcs x23,x23,x0 + mul x0,x8,x4 + adcs x24,x24,x1 + mul x1,x9,x4 + adc x25,x25,x3 + mul x3,x10,x4 + adds x19,x19,x26 + umulh x26,x5,x4 + adcs x20,x20,x27 + umulh x27,x6,x4 + adcs x21,x21,x28 + umulh x28,x7,x4 + adcs x22,x22,x0 + umulh x0,x8,x4 + adcs x23,x23,x1 + umulh x1,x9,x4 + adcs x24,x24,x3 + umulh x3,x10,x4 + adc x25,x25,xzr + ldp x4,x2,[x29,#96] // pull r_ptr + + adds x11,x20,x26 + adcs x12,x21,x27 + adcs x13,x22,x28 + adcs x14,x23,x0 + adcs x15,x24,x1 + adcs x16,x25,x3 + + ret + ENDP + + + + EXPORT |sgn0_pty_mont_384|[FUNC] + ALIGN 32 +|sgn0_pty_mont_384| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + adds x11,x11,x11 + adcs x12,x12,x12 + adcs x13,x13,x13 + adcs x14,x14,x14 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + + + + EXPORT |sgn0_pty_mont_384x|[FUNC] + ALIGN 32 +|sgn0_pty_mont_384x| PROC + DCDU 3573752639 + stp x29,x30,[sp,#-128]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + + mov x4,x2 + ldp x5,x6,[x1] + ldp x7,x8,[x1,#16] + ldp x9,x10,[x1,#32] + mov x1,x0 + + bl __mul_by_1_mont_384 + add x1,x1,#48 + + and x2,x11,#1 + orr x3,x11,x12 + adds x11,x11,x11 + orr x3,x3,x13 + adcs x12,x12,x12 + orr x3,x3,x14 + adcs x13,x13,x13 + orr x3,x3,x15 + adcs x14,x14,x14 + orr x3,x3,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x2,x2,x17 + + bl __mul_by_1_mont_384 + ldr x30,[x29,#8] + + and x0,x11,#1 + orr x1,x11,x12 + adds x11,x11,x11 + orr x1,x1,x13 + adcs x12,x12,x12 + orr x1,x1,x14 + adcs x13,x13,x13 + orr x1,x1,x15 + adcs x14,x14,x14 + orr x1,x1,x16 + adcs x15,x15,x15 + adcs x16,x16,x16 + adc x17,xzr,xzr + + subs x11,x11,x5 + sbcs x12,x12,x6 + sbcs x13,x13,x7 + sbcs x14,x14,x8 + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbc x17,x17,xzr + + mvn x17,x17 + and x17,x17,#2 + orr x0,x0,x17 + + cmp x3,#0 + cseleq x3,x0,x2 + + cmp x1,#0 + cselne x1,x0,x2 + + and x3,x3,#1 + and x1,x1,#2 + orr x0,x1,x3 // pack sign and parity + + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldr x29,[sp],#128 + DCDU 3573752767 + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/mulq_mont_256-x86_64.asm b/crypto/blst_src/build/win64/mulq_mont_256-x86_64.asm new file mode 100644 index 00000000000..c3bf8634617 --- /dev/null +++ b/crypto/blst_src/build/win64/mulq_mont_256-x86_64.asm @@ -0,0 +1,884 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC mul_mont_sparse_256 + + +ALIGN 32 +mul_mont_sparse_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_mont_sparse_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_mul_mont_sparse_256:: + + + mov rax,QWORD PTR[rdx] + mov r13,QWORD PTR[rsi] + mov r14,QWORD PTR[8+rsi] + mov r12,QWORD PTR[16+rsi] + mov rbp,QWORD PTR[24+rsi] + mov rbx,rdx + + mov r15,rax + mul r13 + mov r9,rax + mov rax,r15 + mov r10,rdx + call __mulq_mont_sparse_256 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mul_mont_sparse_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_mont_sparse_256:: +mul_mont_sparse_256 ENDP + +PUBLIC sqr_mont_sparse_256 + + +ALIGN 32 +sqr_mont_sparse_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_mont_sparse_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_sqr_mont_sparse_256:: + + + mov rax,QWORD PTR[rsi] + mov r8,rcx + mov r14,QWORD PTR[8+rsi] + mov rcx,rdx + mov r12,QWORD PTR[16+rsi] + lea rbx,QWORD PTR[rsi] + mov rbp,QWORD PTR[24+rsi] + + mov r15,rax + mul rax + mov r9,rax + mov rax,r15 + mov r10,rdx + call __mulq_mont_sparse_256 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqr_mont_sparse_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_mont_sparse_256:: +sqr_mont_sparse_256 ENDP + +ALIGN 32 +__mulq_mont_sparse_256 PROC PRIVATE + DB 243,15,30,250 + mul r14 + add r10,rax + mov rax,r15 + adc rdx,0 + mov r11,rdx + + mul r12 + add r11,rax + mov rax,r15 + adc rdx,0 + mov r12,rdx + + mul rbp + add r12,rax + mov rax,QWORD PTR[8+rbx] + adc rdx,0 + xor r14,r14 + mov r13,rdx + + mov rdi,r9 + imul r9,r8 + + + mov r15,rax + mul QWORD PTR[rsi] + add r10,rax + mov rax,r15 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[8+rsi] + add r11,rax + mov rax,r15 + adc rdx,0 + add r11,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rsi] + add r12,rax + mov rax,r15 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rsi] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,rbp + adc r14,rdx + xor r15,r15 + + + mul QWORD PTR[rcx] + add rdi,rax + mov rax,r9 + adc rdi,rdx + + mul QWORD PTR[8+rcx] + add r10,rax + mov rax,r9 + adc rdx,0 + add r10,rdi + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r11,rax + mov rax,r9 + adc rdx,0 + add r11,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r12,rax + mov rax,QWORD PTR[16+rbx] + adc rdx,0 + add r12,rbp + adc rdx,0 + add r13,rdx + adc r14,0 + adc r15,0 + mov rdi,r10 + imul r10,r8 + + + mov r9,rax + mul QWORD PTR[rsi] + add r11,rax + mov rax,r9 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[8+rsi] + add r12,rax + mov rax,r9 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rsi] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rsi] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,rbp + adc r15,rdx + xor r9,r9 + + + mul QWORD PTR[rcx] + add rdi,rax + mov rax,r10 + adc rdi,rdx + + mul QWORD PTR[8+rcx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,rdi + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r13,rax + mov rax,QWORD PTR[24+rbx] + adc rdx,0 + add r13,rbp + adc rdx,0 + add r14,rdx + adc r15,0 + adc r9,0 + mov rdi,r11 + imul r11,r8 + + + mov r10,rax + mul QWORD PTR[rsi] + add r12,rax + mov rax,r10 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[8+rsi] + add r13,rax + mov rax,r10 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rsi] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rsi] + add r15,rax + mov rax,r11 + adc rdx,0 + add r15,rbp + adc r9,rdx + xor r10,r10 + + + mul QWORD PTR[rcx] + add rdi,rax + mov rax,r11 + adc rdi,rdx + + mul QWORD PTR[8+rcx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,rdi + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,rbp + adc rdx,0 + add r15,rdx + adc r9,0 + adc r10,0 + imul rax,r8 + mov rsi,QWORD PTR[8+rsp] + + + mov r11,rax + mul QWORD PTR[rcx] + add r12,rax + mov rax,r11 + adc r12,rdx + + mul QWORD PTR[8+rcx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r12 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r14,rax + mov rax,r11 + adc rdx,0 + add r14,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + mov rbx,r14 + add r15,rbp + adc rdx,0 + add r15,rax + mov rax,r13 + adc rdx,0 + add r9,rdx + adc r10,0 + + + + + mov r12,r15 + sub r13,QWORD PTR[rcx] + sbb r14,QWORD PTR[8+rcx] + sbb r15,QWORD PTR[16+rcx] + mov rbp,r9 + sbb r9,QWORD PTR[24+rcx] + sbb r10,0 + + cmovc r13,rax + cmovc r14,rbx + cmovc r15,r12 + mov QWORD PTR[rsi],r13 + cmovc r9,rbp + mov QWORD PTR[8+rsi],r14 + mov QWORD PTR[16+rsi],r15 + mov QWORD PTR[24+rsi],r9 + + DB 0F3h,0C3h ;repret + +__mulq_mont_sparse_256 ENDP +PUBLIC from_mont_256 + + +ALIGN 32 +from_mont_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_from_mont_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_from_mont_256:: + + + mov rbx,rdx + call __mulq_by_1_mont_256 + + + + + + mov r10,r14 + mov r11,r15 + mov r12,r9 + + sub r13,QWORD PTR[rbx] + sbb r14,QWORD PTR[8+rbx] + sbb r15,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + + cmovnc rax,r13 + cmovnc r10,r14 + cmovnc r11,r15 + mov QWORD PTR[rdi],rax + cmovnc r12,r9 + mov QWORD PTR[8+rdi],r10 + mov QWORD PTR[16+rdi],r11 + mov QWORD PTR[24+rdi],r12 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_from_mont_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_from_mont_256:: +from_mont_256 ENDP + +PUBLIC redc_mont_256 + + +ALIGN 32 +redc_mont_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_redc_mont_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_redc_mont_256:: + + + mov rbx,rdx + call __mulq_by_1_mont_256 + + add r13,QWORD PTR[32+rsi] + adc r14,QWORD PTR[40+rsi] + mov rax,r13 + adc r15,QWORD PTR[48+rsi] + mov r10,r14 + adc r9,QWORD PTR[56+rsi] + sbb rsi,rsi + + + + + mov r11,r15 + sub r13,QWORD PTR[rbx] + sbb r14,QWORD PTR[8+rbx] + sbb r15,QWORD PTR[16+rbx] + mov r12,r9 + sbb r9,QWORD PTR[24+rbx] + sbb rsi,0 + + cmovnc rax,r13 + cmovnc r10,r14 + cmovnc r11,r15 + mov QWORD PTR[rdi],rax + cmovnc r12,r9 + mov QWORD PTR[8+rdi],r10 + mov QWORD PTR[16+rdi],r11 + mov QWORD PTR[24+rdi],r12 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_redc_mont_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_redc_mont_256:: +redc_mont_256 ENDP + +ALIGN 32 +__mulq_by_1_mont_256 PROC PRIVATE + DB 243,15,30,250 + mov rax,QWORD PTR[rsi] + mov r10,QWORD PTR[8+rsi] + mov r11,QWORD PTR[16+rsi] + mov r12,QWORD PTR[24+rsi] + + mov r13,rax + imul rax,rcx + mov r9,rax + + mul QWORD PTR[rbx] + add r13,rax + mov rax,r9 + adc r13,rdx + + mul QWORD PTR[8+rbx] + add r10,rax + mov rax,r9 + adc rdx,0 + add r10,r13 + adc rdx,0 + mov r13,rdx + + mul QWORD PTR[16+rbx] + mov r14,r10 + imul r10,rcx + add r11,rax + mov rax,r9 + adc rdx,0 + add r11,r13 + adc rdx,0 + mov r13,rdx + + mul QWORD PTR[24+rbx] + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,r13 + adc rdx,0 + mov r13,rdx + + mul QWORD PTR[rbx] + add r14,rax + mov rax,r10 + adc r14,rdx + + mul QWORD PTR[8+rbx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[16+rbx] + mov r15,r11 + imul r11,rcx + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[24+rbx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[rbx] + add r15,rax + mov rax,r11 + adc r15,rdx + + mul QWORD PTR[8+rbx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[16+rbx] + mov r9,r12 + imul r12,rcx + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[24+rbx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[rbx] + add r9,rax + mov rax,r12 + adc r9,rdx + + mul QWORD PTR[8+rbx] + add r13,rax + mov rax,r12 + adc rdx,0 + add r13,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[16+rbx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rbx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,r9 + adc rdx,0 + mov r9,rdx + DB 0F3h,0C3h ;repret +__mulq_by_1_mont_256 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mul_mont_sparse_256 + DD imagerel $L$SEH_body_mul_mont_sparse_256 + DD imagerel $L$SEH_info_mul_mont_sparse_256_prologue + + DD imagerel $L$SEH_body_mul_mont_sparse_256 + DD imagerel $L$SEH_epilogue_mul_mont_sparse_256 + DD imagerel $L$SEH_info_mul_mont_sparse_256_body + + DD imagerel $L$SEH_epilogue_mul_mont_sparse_256 + DD imagerel $L$SEH_end_mul_mont_sparse_256 + DD imagerel $L$SEH_info_mul_mont_sparse_256_epilogue + + DD imagerel $L$SEH_begin_sqr_mont_sparse_256 + DD imagerel $L$SEH_body_sqr_mont_sparse_256 + DD imagerel $L$SEH_info_sqr_mont_sparse_256_prologue + + DD imagerel $L$SEH_body_sqr_mont_sparse_256 + DD imagerel $L$SEH_epilogue_sqr_mont_sparse_256 + DD imagerel $L$SEH_info_sqr_mont_sparse_256_body + + DD imagerel $L$SEH_epilogue_sqr_mont_sparse_256 + DD imagerel $L$SEH_end_sqr_mont_sparse_256 + DD imagerel $L$SEH_info_sqr_mont_sparse_256_epilogue + + DD imagerel $L$SEH_begin_from_mont_256 + DD imagerel $L$SEH_body_from_mont_256 + DD imagerel $L$SEH_info_from_mont_256_prologue + + DD imagerel $L$SEH_body_from_mont_256 + DD imagerel $L$SEH_epilogue_from_mont_256 + DD imagerel $L$SEH_info_from_mont_256_body + + DD imagerel $L$SEH_epilogue_from_mont_256 + DD imagerel $L$SEH_end_from_mont_256 + DD imagerel $L$SEH_info_from_mont_256_epilogue + + DD imagerel $L$SEH_begin_redc_mont_256 + DD imagerel $L$SEH_body_redc_mont_256 + DD imagerel $L$SEH_info_redc_mont_256_prologue + + DD imagerel $L$SEH_body_redc_mont_256 + DD imagerel $L$SEH_epilogue_redc_mont_256 + DD imagerel $L$SEH_info_redc_mont_256_body + + DD imagerel $L$SEH_epilogue_redc_mont_256 + DD imagerel $L$SEH_end_redc_mont_256 + DD imagerel $L$SEH_info_redc_mont_256_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_mul_mont_sparse_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_mont_sparse_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mul_mont_sparse_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_mont_sparse_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_mont_sparse_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqr_mont_sparse_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_from_mont_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_from_mont_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_from_mont_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_redc_mont_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_redc_mont_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_redc_mont_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/mulq_mont_384-x86_64.asm b/crypto/blst_src/build/win64/mulq_mont_384-x86_64.asm new file mode 100644 index 00000000000..0ccb46786c3 --- /dev/null +++ b/crypto/blst_src/build/win64/mulq_mont_384-x86_64.asm @@ -0,0 +1,4233 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + + + + + + + + +ALIGN 32 +__sub_mod_384x384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + + sub r8,QWORD PTR[rdx] + mov r15,QWORD PTR[56+rsi] + sbb r9,QWORD PTR[8+rdx] + mov rax,QWORD PTR[64+rsi] + sbb r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[72+rsi] + sbb r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[80+rsi] + sbb r12,QWORD PTR[32+rdx] + mov rsi,QWORD PTR[88+rsi] + sbb r13,QWORD PTR[40+rdx] + mov QWORD PTR[rdi],r8 + sbb r14,QWORD PTR[48+rdx] + mov r8,QWORD PTR[rcx] + mov QWORD PTR[8+rdi],r9 + sbb r15,QWORD PTR[56+rdx] + mov r9,QWORD PTR[8+rcx] + mov QWORD PTR[16+rdi],r10 + sbb rax,QWORD PTR[64+rdx] + mov r10,QWORD PTR[16+rcx] + mov QWORD PTR[24+rdi],r11 + sbb rbx,QWORD PTR[72+rdx] + mov r11,QWORD PTR[24+rcx] + mov QWORD PTR[32+rdi],r12 + sbb rbp,QWORD PTR[80+rdx] + mov r12,QWORD PTR[32+rcx] + mov QWORD PTR[40+rdi],r13 + sbb rsi,QWORD PTR[88+rdx] + mov r13,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r8,rdx + and r9,rdx + and r10,rdx + and r11,rdx + and r12,rdx + and r13,rdx + + add r14,r8 + adc r15,r9 + mov QWORD PTR[48+rdi],r14 + adc rax,r10 + mov QWORD PTR[56+rdi],r15 + adc rbx,r11 + mov QWORD PTR[64+rdi],rax + adc rbp,r12 + mov QWORD PTR[72+rdi],rbx + adc rsi,r13 + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rsi + + DB 0F3h,0C3h ;repret +__sub_mod_384x384 ENDP + + +ALIGN 32 +__add_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + adc r10,QWORD PTR[16+rdx] + mov r14,r8 + adc r11,QWORD PTR[24+rdx] + mov r15,r9 + adc r12,QWORD PTR[32+rdx] + mov rax,r10 + adc r13,QWORD PTR[40+rdx] + mov rbx,r11 + sbb rdx,rdx + + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + mov rbp,r12 + sbb r10,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rcx] + mov rsi,r13 + sbb r13,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + mov QWORD PTR[rdi],r8 + cmovc r11,rbx + mov QWORD PTR[8+rdi],r9 + cmovc r12,rbp + mov QWORD PTR[16+rdi],r10 + cmovc r13,rsi + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__add_mod_384 ENDP + + +ALIGN 32 +__sub_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + +__sub_mod_384_a_is_loaded:: + sub r8,QWORD PTR[rdx] + mov r14,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rdx] + mov r15,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rdx] + mov rax,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rdx] + mov rbx,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rdx] + mov rbp,QWORD PTR[32+rcx] + sbb r13,QWORD PTR[40+rdx] + mov rsi,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r14,rdx + and r15,rdx + and rax,rdx + and rbx,rdx + and rbp,rdx + and rsi,rdx + + add r8,r14 + adc r9,r15 + mov QWORD PTR[rdi],r8 + adc r10,rax + mov QWORD PTR[8+rdi],r9 + adc r11,rbx + mov QWORD PTR[16+rdi],r10 + adc r12,rbp + mov QWORD PTR[24+rdi],r11 + adc r13,rsi + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__sub_mod_384 ENDP +PUBLIC mul_mont_384x + + +ALIGN 32 +mul_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,328 + +$L$SEH_body_mul_mont_384x:: + + + mov rbx,rdx + mov QWORD PTR[32+rsp],rdi + mov QWORD PTR[24+rsp],rsi + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[8+rsp],rcx + mov QWORD PTR[rsp],r8 + + + + + lea rdi,QWORD PTR[40+rsp] + call __mulq_384 + + + lea rbx,QWORD PTR[48+rbx] + lea rsi,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[((40+96))+rsp] + call __mulq_384 + + + mov rcx,QWORD PTR[8+rsp] + lea rdx,QWORD PTR[((-48))+rsi] + lea rdi,QWORD PTR[((40+192+48))+rsp] + call __add_mod_384 + + mov rsi,QWORD PTR[16+rsp] + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[((-48))+rdi] + call __add_mod_384 + + lea rbx,QWORD PTR[rdi] + lea rsi,QWORD PTR[48+rdi] + call __mulq_384 + + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[40+rsp] + mov rcx,QWORD PTR[8+rsp] + call __sub_mod_384x384 + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[40+rsp] + lea rdx,QWORD PTR[((40+96))+rsp] + lea rdi,QWORD PTR[40+rsp] + call __sub_mod_384x384 + + mov rbx,rcx + + + lea rsi,QWORD PTR[40+rsp] + mov rcx,QWORD PTR[rsp] + mov rdi,QWORD PTR[32+rsp] + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + + lea rsi,QWORD PTR[((40+192))+rsp] + mov rcx,QWORD PTR[rsp] + lea rdi,QWORD PTR[48+rdi] + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + lea r8,QWORD PTR[328+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_mul_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_mont_384x:: +mul_mont_384x ENDP +PUBLIC sqr_mont_384x + + +ALIGN 32 +sqr_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_sqr_mont_384x:: + + + mov QWORD PTR[rsp],rcx + mov rcx,rdx + mov QWORD PTR[8+rsp],rdi + mov QWORD PTR[16+rsp],rsi + + + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[32+rsp] + call __add_mod_384 + + + mov rsi,QWORD PTR[16+rsp] + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[((32+48))+rsp] + call __sub_mod_384 + + + mov rsi,QWORD PTR[16+rsp] + lea rbx,QWORD PTR[48+rsi] + + mov rax,QWORD PTR[48+rsi] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov r12,QWORD PTR[16+rsi] + mov r13,QWORD PTR[24+rsi] + + call __mulq_mont_384 + add r14,r14 + adc r15,r15 + adc r8,r8 + mov r12,r14 + adc r9,r9 + mov r13,r15 + adc r10,r10 + mov rax,r8 + adc r11,r11 + mov rbx,r9 + sbb rdx,rdx + + sub r14,QWORD PTR[rcx] + sbb r15,QWORD PTR[8+rcx] + mov rbp,r10 + sbb r8,QWORD PTR[16+rcx] + sbb r9,QWORD PTR[24+rcx] + sbb r10,QWORD PTR[32+rcx] + mov rsi,r11 + sbb r11,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r14,r12 + cmovc r15,r13 + cmovc r8,rax + mov QWORD PTR[48+rdi],r14 + cmovc r9,rbx + mov QWORD PTR[56+rdi],r15 + cmovc r10,rbp + mov QWORD PTR[64+rdi],r8 + cmovc r11,rsi + mov QWORD PTR[72+rdi],r9 + mov QWORD PTR[80+rdi],r10 + mov QWORD PTR[88+rdi],r11 + + lea rsi,QWORD PTR[32+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + + mov rax,QWORD PTR[((32+48))+rsp] + mov r14,QWORD PTR[((32+0))+rsp] + mov r15,QWORD PTR[((32+8))+rsp] + mov r12,QWORD PTR[((32+16))+rsp] + mov r13,QWORD PTR[((32+24))+rsp] + + call __mulq_mont_384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqr_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_mont_384x:: +sqr_mont_384x ENDP + +PUBLIC mul_382x + + +ALIGN 32 +mul_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_mul_382x:: + + + lea rdi,QWORD PTR[96+rdi] + mov QWORD PTR[rsp],rsi + mov QWORD PTR[8+rsp],rdx + mov QWORD PTR[16+rsp],rdi + mov QWORD PTR[24+rsp],rcx + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + add r8,QWORD PTR[48+rsi] + adc r9,QWORD PTR[56+rsi] + adc r10,QWORD PTR[64+rsi] + adc r11,QWORD PTR[72+rsi] + adc r12,QWORD PTR[80+rsi] + adc r13,QWORD PTR[88+rsi] + + mov QWORD PTR[((32+0))+rsp],r8 + mov QWORD PTR[((32+8))+rsp],r9 + mov QWORD PTR[((32+16))+rsp],r10 + mov QWORD PTR[((32+24))+rsp],r11 + mov QWORD PTR[((32+32))+rsp],r12 + mov QWORD PTR[((32+40))+rsp],r13 + + + mov r8,QWORD PTR[rdx] + mov r9,QWORD PTR[8+rdx] + mov r10,QWORD PTR[16+rdx] + mov r11,QWORD PTR[24+rdx] + mov r12,QWORD PTR[32+rdx] + mov r13,QWORD PTR[40+rdx] + + add r8,QWORD PTR[48+rdx] + adc r9,QWORD PTR[56+rdx] + adc r10,QWORD PTR[64+rdx] + adc r11,QWORD PTR[72+rdx] + adc r12,QWORD PTR[80+rdx] + adc r13,QWORD PTR[88+rdx] + + mov QWORD PTR[((32+48))+rsp],r8 + mov QWORD PTR[((32+56))+rsp],r9 + mov QWORD PTR[((32+64))+rsp],r10 + mov QWORD PTR[((32+72))+rsp],r11 + mov QWORD PTR[((32+80))+rsp],r12 + mov QWORD PTR[((32+88))+rsp],r13 + + + lea rsi,QWORD PTR[((32+0))+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + call __mulq_384 + + + mov rsi,QWORD PTR[rsp] + mov rbx,QWORD PTR[8+rsp] + lea rdi,QWORD PTR[((-96))+rdi] + call __mulq_384 + + + lea rsi,QWORD PTR[48+rsi] + lea rbx,QWORD PTR[48+rbx] + lea rdi,QWORD PTR[32+rsp] + call __mulq_384 + + + mov rsi,QWORD PTR[16+rsp] + lea rdx,QWORD PTR[32+rsp] + mov rcx,QWORD PTR[24+rsp] + mov rdi,rsi + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[((-96))+rdi] + lea rdx,QWORD PTR[32+rsp] + lea rdi,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_mul_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_382x:: +mul_382x ENDP +PUBLIC sqr_382x + + +ALIGN 32 +sqr_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rsi + +$L$SEH_body_sqr_382x:: + + + mov rcx,rdx + + + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov rbx,QWORD PTR[24+rsi] + mov rbp,QWORD PTR[32+rsi] + mov rdx,QWORD PTR[40+rsi] + + mov r8,r14 + add r14,QWORD PTR[48+rsi] + mov r9,r15 + adc r15,QWORD PTR[56+rsi] + mov r10,rax + adc rax,QWORD PTR[64+rsi] + mov r11,rbx + adc rbx,QWORD PTR[72+rsi] + mov r12,rbp + adc rbp,QWORD PTR[80+rsi] + mov r13,rdx + adc rdx,QWORD PTR[88+rsi] + + mov QWORD PTR[rdi],r14 + mov QWORD PTR[8+rdi],r15 + mov QWORD PTR[16+rdi],rax + mov QWORD PTR[24+rdi],rbx + mov QWORD PTR[32+rdi],rbp + mov QWORD PTR[40+rdi],rdx + + + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[48+rdi] + call __sub_mod_384_a_is_loaded + + + lea rsi,QWORD PTR[rdi] + lea rbx,QWORD PTR[((-48))+rdi] + lea rdi,QWORD PTR[((-48))+rdi] + call __mulq_384 + + + mov rsi,QWORD PTR[rsp] + lea rbx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[96+rdi] + call __mulq_384 + + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov r12,QWORD PTR[32+rdi] + mov r13,QWORD PTR[40+rdi] + mov r14,QWORD PTR[48+rdi] + mov r15,QWORD PTR[56+rdi] + mov rax,QWORD PTR[64+rdi] + mov rbx,QWORD PTR[72+rdi] + mov rbp,QWORD PTR[80+rdi] + add r8,r8 + mov rdx,QWORD PTR[88+rdi] + adc r9,r9 + mov QWORD PTR[rdi],r8 + adc r10,r10 + mov QWORD PTR[8+rdi],r9 + adc r11,r11 + mov QWORD PTR[16+rdi],r10 + adc r12,r12 + mov QWORD PTR[24+rdi],r11 + adc r13,r13 + mov QWORD PTR[32+rdi],r12 + adc r14,r14 + mov QWORD PTR[40+rdi],r13 + adc r15,r15 + mov QWORD PTR[48+rdi],r14 + adc rax,rax + mov QWORD PTR[56+rdi],r15 + adc rbx,rbx + mov QWORD PTR[64+rdi],rax + adc rbp,rbp + mov QWORD PTR[72+rdi],rbx + adc rdx,rdx + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rdx + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqr_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_382x:: +sqr_382x ENDP +PUBLIC mul_384 + + +ALIGN 32 +mul_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + +$L$SEH_body_mul_384:: + + + mov rbx,rdx + call __mulq_384 + + mov r12,QWORD PTR[rsp] + + mov rbx,QWORD PTR[8+rsp] + + mov rbp,QWORD PTR[16+rsp] + + lea rsp,QWORD PTR[24+rsp] + +$L$SEH_epilogue_mul_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_384:: +mul_384 ENDP + + +ALIGN 32 +__mulq_384 PROC PRIVATE + DB 243,15,30,250 + mov rax,QWORD PTR[rbx] + + mov rbp,rax + mul QWORD PTR[rsi] + mov QWORD PTR[rdi],rax + mov rax,rbp + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r11,rax + mov rax,QWORD PTR[8+rbx] + adc rdx,0 + mov r12,rdx + mov rbp,rax + mul QWORD PTR[rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov QWORD PTR[8+rdi],rcx + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r12,rax + mov rax,QWORD PTR[16+rbx] + adc rdx,0 + add r11,r12 + adc rdx,0 + mov r12,rdx + mov rbp,rax + mul QWORD PTR[rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov QWORD PTR[16+rdi],rcx + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r12,rax + mov rax,QWORD PTR[24+rbx] + adc rdx,0 + add r11,r12 + adc rdx,0 + mov r12,rdx + mov rbp,rax + mul QWORD PTR[rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov QWORD PTR[24+rdi],rcx + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r12,rax + mov rax,QWORD PTR[32+rbx] + adc rdx,0 + add r11,r12 + adc rdx,0 + mov r12,rdx + mov rbp,rax + mul QWORD PTR[rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov QWORD PTR[32+rdi],rcx + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r12,rax + mov rax,QWORD PTR[40+rbx] + adc rdx,0 + add r11,r12 + adc rdx,0 + mov r12,rdx + mov rbp,rax + mul QWORD PTR[rsi] + add rcx,rax + mov rax,rbp + adc rdx,0 + mov QWORD PTR[40+rdi],rcx + mov rcx,rdx + + mul QWORD PTR[8+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r12,rax + mov rax,rax + adc rdx,0 + add r11,r12 + adc rdx,0 + mov r12,rdx + mov QWORD PTR[48+rdi],rcx + mov QWORD PTR[56+rdi],r8 + mov QWORD PTR[64+rdi],r9 + mov QWORD PTR[72+rdi],r10 + mov QWORD PTR[80+rdi],r11 + mov QWORD PTR[88+rdi],r12 + + DB 0F3h,0C3h ;repret +__mulq_384 ENDP +PUBLIC sqr_384 + + +ALIGN 32 +sqr_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_384:: + mov rdi,rcx + mov rsi,rdx + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sqr_384:: + + + call __sqrq_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqr_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_384:: +sqr_384 ENDP + + +ALIGN 32 +__sqrq_384 PROC PRIVATE + DB 243,15,30,250 + mov rax,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rcx,QWORD PTR[16+rsi] + mov rbx,QWORD PTR[24+rsi] + + + mov r14,rax + mul r15 + mov r9,rax + mov rax,r14 + mov rbp,QWORD PTR[32+rsi] + mov r10,rdx + + mul rcx + add r10,rax + mov rax,r14 + adc rdx,0 + mov rsi,QWORD PTR[40+rsi] + mov r11,rdx + + mul rbx + add r11,rax + mov rax,r14 + adc rdx,0 + mov r12,rdx + + mul rbp + add r12,rax + mov rax,r14 + adc rdx,0 + mov r13,rdx + + mul rsi + add r13,rax + mov rax,r14 + adc rdx,0 + mov r14,rdx + + mul rax + xor r8,r8 + mov QWORD PTR[rdi],rax + mov rax,r15 + add r9,r9 + adc r8,0 + add r9,rdx + adc r8,0 + mov QWORD PTR[8+rdi],r9 + + mul rcx + add r11,rax + mov rax,r15 + adc rdx,0 + mov r9,rdx + + mul rbx + add r12,rax + mov rax,r15 + adc rdx,0 + add r12,r9 + adc rdx,0 + mov r9,rdx + + mul rbp + add r13,rax + mov rax,r15 + adc rdx,0 + add r13,r9 + adc rdx,0 + mov r9,rdx + + mul rsi + add r14,rax + mov rax,r15 + adc rdx,0 + add r14,r9 + adc rdx,0 + mov r15,rdx + + mul rax + xor r9,r9 + add r8,rax + mov rax,rcx + add r10,r10 + adc r11,r11 + adc r9,0 + add r10,r8 + adc r11,rdx + adc r9,0 + mov QWORD PTR[16+rdi],r10 + + mul rbx + add r13,rax + mov rax,rcx + adc rdx,0 + mov QWORD PTR[24+rdi],r11 + mov r8,rdx + + mul rbp + add r14,rax + mov rax,rcx + adc rdx,0 + add r14,r8 + adc rdx,0 + mov r8,rdx + + mul rsi + add r15,rax + mov rax,rcx + adc rdx,0 + add r15,r8 + adc rdx,0 + mov rcx,rdx + + mul rax + xor r11,r11 + add r9,rax + mov rax,rbx + add r12,r12 + adc r13,r13 + adc r11,0 + add r12,r9 + adc r13,rdx + adc r11,0 + mov QWORD PTR[32+rdi],r12 + + + mul rbp + add r15,rax + mov rax,rbx + adc rdx,0 + mov QWORD PTR[40+rdi],r13 + mov r8,rdx + + mul rsi + add rcx,rax + mov rax,rbx + adc rdx,0 + add rcx,r8 + adc rdx,0 + mov rbx,rdx + + mul rax + xor r12,r12 + add r11,rax + mov rax,rbp + add r14,r14 + adc r15,r15 + adc r12,0 + add r14,r11 + adc r15,rdx + mov QWORD PTR[48+rdi],r14 + adc r12,0 + mov QWORD PTR[56+rdi],r15 + + + mul rsi + add rbx,rax + mov rax,rbp + adc rdx,0 + mov rbp,rdx + + mul rax + xor r13,r13 + add r12,rax + mov rax,rsi + add rcx,rcx + adc rbx,rbx + adc r13,0 + add rcx,r12 + adc rbx,rdx + mov QWORD PTR[64+rdi],rcx + adc r13,0 + mov QWORD PTR[72+rdi],rbx + + + mul rax + add rax,r13 + add rbp,rbp + adc rdx,0 + add rax,rbp + adc rdx,0 + mov QWORD PTR[80+rdi],rax + mov QWORD PTR[88+rdi],rdx + + DB 0F3h,0C3h ;repret +__sqrq_384 ENDP + +PUBLIC sqr_mont_384 + + +ALIGN 32 +sqr_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8*15 + +$L$SEH_body_sqr_mont_384:: + + + mov QWORD PTR[96+rsp],rcx + mov QWORD PTR[104+rsp],rdx + mov QWORD PTR[112+rsp],rdi + + mov rdi,rsp + call __sqrq_384 + + lea rsi,QWORD PTR[rsp] + mov rcx,QWORD PTR[96+rsp] + mov rbx,QWORD PTR[104+rsp] + mov rdi,QWORD PTR[112+rsp] + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + lea r8,QWORD PTR[120+rsp] + mov r15,QWORD PTR[120+rsp] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqr_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_mont_384:: +sqr_mont_384 ENDP + + + +PUBLIC redc_mont_384 + + +ALIGN 32 +redc_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_redc_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_redc_mont_384:: + + + mov rbx,rdx + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_redc_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_redc_mont_384:: +redc_mont_384 ENDP + + + + +PUBLIC from_mont_384 + + +ALIGN 32 +from_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_from_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_from_mont_384:: + + + mov rbx,rdx + call __mulq_by_1_mont_384 + + + + + + mov rcx,r15 + mov rdx,r8 + mov rbp,r9 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + mov r13,r10 + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + mov rsi,r11 + sbb r11,QWORD PTR[40+rbx] + + cmovc r14,rax + cmovc r15,rcx + cmovc r8,rdx + mov QWORD PTR[rdi],r14 + cmovc r9,rbp + mov QWORD PTR[8+rdi],r15 + cmovc r10,r13 + mov QWORD PTR[16+rdi],r8 + cmovc r11,rsi + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_from_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_from_mont_384:: +from_mont_384 ENDP + +ALIGN 32 +__mulq_by_1_mont_384 PROC PRIVATE + DB 243,15,30,250 + mov rax,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,rax + imul rax,rcx + mov r8,rax + + mul QWORD PTR[rbx] + add r14,rax + mov rax,r8 + adc r14,rdx + + mul QWORD PTR[8+rbx] + add r9,rax + mov rax,r8 + adc rdx,0 + add r9,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[16+rbx] + add r10,rax + mov rax,r8 + adc rdx,0 + add r10,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[24+rbx] + add r11,rax + mov rax,r8 + adc rdx,0 + mov r15,r9 + imul r9,rcx + add r11,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[32+rbx] + add r12,rax + mov rax,r8 + adc rdx,0 + add r12,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[40+rbx] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[rbx] + add r15,rax + mov rax,r9 + adc r15,rdx + + mul QWORD PTR[8+rbx] + add r10,rax + mov rax,r9 + adc rdx,0 + add r10,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[16+rbx] + add r11,rax + mov rax,r9 + adc rdx,0 + add r11,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[24+rbx] + add r12,rax + mov rax,r9 + adc rdx,0 + mov r8,r10 + imul r10,rcx + add r12,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[32+rbx] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[40+rbx] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[rbx] + add r8,rax + mov rax,r10 + adc r8,rdx + + mul QWORD PTR[8+rbx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rbx] + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[24+rbx] + add r13,rax + mov rax,r10 + adc rdx,0 + mov r9,r11 + imul r11,rcx + add r13,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[32+rbx] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[40+rbx] + add r15,rax + mov rax,r11 + adc rdx,0 + add r15,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[rbx] + add r9,rax + mov rax,r11 + adc r9,rdx + + mul QWORD PTR[8+rbx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[16+rbx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rbx] + add r14,rax + mov rax,r11 + adc rdx,0 + mov r10,r12 + imul r12,rcx + add r14,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[32+rbx] + add r15,rax + mov rax,r11 + adc rdx,0 + add r15,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[40+rbx] + add r8,rax + mov rax,r12 + adc rdx,0 + add r8,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[rbx] + add r10,rax + mov rax,r12 + adc r10,rdx + + mul QWORD PTR[8+rbx] + add r13,rax + mov rax,r12 + adc rdx,0 + add r13,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[16+rbx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[24+rbx] + add r15,rax + mov rax,r12 + adc rdx,0 + mov r11,r13 + imul r13,rcx + add r15,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rbx] + add r8,rax + mov rax,r12 + adc rdx,0 + add r8,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[40+rbx] + add r9,rax + mov rax,r13 + adc rdx,0 + add r9,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[rbx] + add r11,rax + mov rax,r13 + adc r11,rdx + + mul QWORD PTR[8+rbx] + add r14,rax + mov rax,r13 + adc rdx,0 + add r14,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[16+rbx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[24+rbx] + add r8,rax + mov rax,r13 + adc rdx,0 + add r8,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[32+rbx] + add r9,rax + mov rax,r13 + adc rdx,0 + add r9,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rbx] + add r10,rax + mov rax,r14 + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + DB 0F3h,0C3h ;repret +__mulq_by_1_mont_384 ENDP + + +ALIGN 32 +__redc_tail_mont_384 PROC PRIVATE + DB 243,15,30,250 + add r14,QWORD PTR[48+rsi] + mov rax,r14 + adc r15,QWORD PTR[56+rsi] + adc r8,QWORD PTR[64+rsi] + adc r9,QWORD PTR[72+rsi] + mov rcx,r15 + adc r10,QWORD PTR[80+rsi] + adc r11,QWORD PTR[88+rsi] + sbb r12,r12 + + + + + mov rdx,r8 + mov rbp,r9 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + mov r13,r10 + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + mov rsi,r11 + sbb r11,QWORD PTR[40+rbx] + sbb r12,0 + + cmovc r14,rax + cmovc r15,rcx + cmovc r8,rdx + mov QWORD PTR[rdi],r14 + cmovc r9,rbp + mov QWORD PTR[8+rdi],r15 + cmovc r10,r13 + mov QWORD PTR[16+rdi],r8 + cmovc r11,rsi + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + DB 0F3h,0C3h ;repret +__redc_tail_mont_384 ENDP + +PUBLIC sgn0_pty_mont_384 + + +ALIGN 32 +sgn0_pty_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0_pty_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sgn0_pty_mont_384:: + + + mov rbx,rsi + lea rsi,QWORD PTR[rdi] + mov rcx,rdx + call __mulq_by_1_mont_384 + + xor rax,rax + mov r13,r14 + add r14,r14 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rax,0 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rax,0 + + not rax + and r13,1 + and rax,2 + or rax,r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sgn0_pty_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0_pty_mont_384:: +sgn0_pty_mont_384 ENDP + +PUBLIC sgn0_pty_mont_384x + + +ALIGN 32 +sgn0_pty_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0_pty_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sgn0_pty_mont_384x:: + + + mov rbx,rsi + lea rsi,QWORD PTR[48+rdi] + mov rcx,rdx + call __mulq_by_1_mont_384 + + mov r12,r14 + or r14,r15 + or r14,r8 + or r14,r9 + or r14,r10 + or r14,r11 + + lea rsi,QWORD PTR[rdi] + xor rdi,rdi + mov r13,r12 + add r12,r12 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rdi,0 + + sub r12,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rdi,0 + + mov QWORD PTR[rsp],r14 + not rdi + and r13,1 + and rdi,2 + or rdi,r13 + + call __mulq_by_1_mont_384 + + mov r12,r14 + or r14,r15 + or r14,r8 + or r14,r9 + or r14,r10 + or r14,r11 + + xor rax,rax + mov r13,r12 + add r12,r12 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rax,0 + + sub r12,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rax,0 + + mov r12,QWORD PTR[rsp] + + not rax + + test r14,r14 + cmovz r13,rdi + + test r12,r12 + cmovnz rax,rdi + + and r13,1 + and rax,2 + or rax,r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sgn0_pty_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0_pty_mont_384x:: +sgn0_pty_mont_384x ENDP +PUBLIC mul_mont_384 + + +ALIGN 32 +mul_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mul_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8*3 + +$L$SEH_body_mul_mont_384:: + + + mov rax,QWORD PTR[rdx] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov r12,QWORD PTR[16+rsi] + mov r13,QWORD PTR[24+rsi] + mov rbx,rdx + mov QWORD PTR[rsp],r8 + mov QWORD PTR[8+rsp],rdi + + call __mulq_mont_384 + + mov r15,QWORD PTR[24+rsp] + + mov r14,QWORD PTR[32+rsp] + + mov r13,QWORD PTR[40+rsp] + + mov r12,QWORD PTR[48+rsp] + + mov rbx,QWORD PTR[56+rsp] + + mov rbp,QWORD PTR[64+rsp] + + lea rsp,QWORD PTR[72+rsp] + +$L$SEH_epilogue_mul_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mul_mont_384:: +mul_mont_384 ENDP + +ALIGN 32 +__mulq_mont_384 PROC PRIVATE + DB 243,15,30,250 + mov rdi,rax + mul r14 + mov r8,rax + mov rax,rdi + mov r9,rdx + + mul r15 + add r9,rax + mov rax,rdi + adc rdx,0 + mov r10,rdx + + mul r12 + add r10,rax + mov rax,rdi + adc rdx,0 + mov r11,rdx + + mov rbp,r8 + imul r8,QWORD PTR[8+rsp] + + mul r13 + add r11,rax + mov rax,rdi + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[32+rsi] + add r12,rax + mov rax,rdi + adc rdx,0 + mov r13,rdx + + mul QWORD PTR[40+rsi] + add r13,rax + mov rax,r8 + adc rdx,0 + xor r15,r15 + mov r14,rdx + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r8 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r9,rax + mov rax,r8 + adc rdx,0 + add r9,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r10,rax + mov rax,r8 + adc rdx,0 + add r10,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r11,rbp + adc rdx,0 + add r11,rax + mov rax,r8 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r12,rax + mov rax,r8 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r13,rax + mov rax,QWORD PTR[8+rbx] + adc rdx,0 + add r13,rbp + adc r14,rdx + adc r15,0 + + mov rdi,rax + mul QWORD PTR[rsi] + add r9,rax + mov rax,rdi + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[8+rsi] + add r10,rax + mov rax,rdi + adc rdx,0 + add r10,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r11,rax + mov rax,rdi + adc rdx,0 + add r11,r8 + adc rdx,0 + mov r8,rdx + + mov rbp,r9 + imul r9,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r12,rax + mov rax,rdi + adc rdx,0 + add r12,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[32+rsi] + add r13,rax + mov rax,rdi + adc rdx,0 + add r13,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[40+rsi] + add r14,r8 + adc rdx,0 + xor r8,r8 + add r14,rax + mov rax,r9 + adc r15,rdx + adc r8,0 + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r9 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r10,rax + mov rax,r9 + adc rdx,0 + add r10,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r11,rax + mov rax,r9 + adc rdx,0 + add r11,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r12,rbp + adc rdx,0 + add r12,rax + mov rax,r9 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r14,rax + mov rax,QWORD PTR[16+rbx] + adc rdx,0 + add r14,rbp + adc r15,rdx + adc r8,0 + + mov rdi,rax + mul QWORD PTR[rsi] + add r10,rax + mov rax,rdi + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[8+rsi] + add r11,rax + mov rax,rdi + adc rdx,0 + add r11,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[16+rsi] + add r12,rax + mov rax,rdi + adc rdx,0 + add r12,r9 + adc rdx,0 + mov r9,rdx + + mov rbp,r10 + imul r10,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r13,rax + mov rax,rdi + adc rdx,0 + add r13,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[32+rsi] + add r14,rax + mov rax,rdi + adc rdx,0 + add r14,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[40+rsi] + add r15,r9 + adc rdx,0 + xor r9,r9 + add r15,rax + mov rax,r10 + adc r8,rdx + adc r9,0 + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r10 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r13,rbp + adc rdx,0 + add r13,rax + mov rax,r10 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r15,rax + mov rax,QWORD PTR[24+rbx] + adc rdx,0 + add r15,rbp + adc r8,rdx + adc r9,0 + + mov rdi,rax + mul QWORD PTR[rsi] + add r11,rax + mov rax,rdi + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[8+rsi] + add r12,rax + mov rax,rdi + adc rdx,0 + add r12,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[16+rsi] + add r13,rax + mov rax,rdi + adc rdx,0 + add r13,r10 + adc rdx,0 + mov r10,rdx + + mov rbp,r11 + imul r11,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r14,rax + mov rax,rdi + adc rdx,0 + add r14,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r15,rax + mov rax,rdi + adc rdx,0 + add r15,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[40+rsi] + add r8,r10 + adc rdx,0 + xor r10,r10 + add r8,rax + mov rax,r11 + adc r9,rdx + adc r10,0 + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r11 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r14,rbp + adc rdx,0 + add r14,rax + mov rax,r11 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r15,rax + mov rax,r11 + adc rdx,0 + add r15,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r8,rax + mov rax,QWORD PTR[32+rbx] + adc rdx,0 + add r8,rbp + adc r9,rdx + adc r10,0 + + mov rdi,rax + mul QWORD PTR[rsi] + add r12,rax + mov rax,rdi + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[8+rsi] + add r13,rax + mov rax,rdi + adc rdx,0 + add r13,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[16+rsi] + add r14,rax + mov rax,rdi + adc rdx,0 + add r14,r11 + adc rdx,0 + mov r11,rdx + + mov rbp,r12 + imul r12,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r15,rax + mov rax,rdi + adc rdx,0 + add r15,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[32+rsi] + add r8,rax + mov rax,rdi + adc rdx,0 + add r8,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r9,r11 + adc rdx,0 + xor r11,r11 + add r9,rax + mov rax,r12 + adc r10,rdx + adc r11,0 + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r12 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r13,rax + mov rax,r12 + adc rdx,0 + add r13,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r15,rbp + adc rdx,0 + add r15,rax + mov rax,r12 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r8,rax + mov rax,r12 + adc rdx,0 + add r8,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r9,rax + mov rax,QWORD PTR[40+rbx] + adc rdx,0 + add r9,rbp + adc r10,rdx + adc r11,0 + + mov rdi,rax + mul QWORD PTR[rsi] + add r13,rax + mov rax,rdi + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[8+rsi] + add r14,rax + mov rax,rdi + adc rdx,0 + add r14,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[16+rsi] + add r15,rax + mov rax,rdi + adc rdx,0 + add r15,r12 + adc rdx,0 + mov r12,rdx + + mov rbp,r13 + imul r13,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r8,rax + mov rax,rdi + adc rdx,0 + add r8,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[32+rsi] + add r9,rax + mov rax,rdi + adc rdx,0 + add r9,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[40+rsi] + add r10,r12 + adc rdx,0 + xor r12,r12 + add r10,rax + mov rax,r13 + adc r11,rdx + adc r12,0 + + mul QWORD PTR[rcx] + add rbp,rax + mov rax,r13 + adc rbp,rdx + + mul QWORD PTR[8+rcx] + add r14,rax + mov rax,r13 + adc rdx,0 + add r14,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[16+rcx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[24+rcx] + add r8,rbp + adc rdx,0 + add r8,rax + mov rax,r13 + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[32+rcx] + add r9,rax + mov rax,r13 + adc rdx,0 + add r9,rbp + adc rdx,0 + mov rbp,rdx + + mul QWORD PTR[40+rcx] + add r10,rax + mov rax,r14 + adc rdx,0 + add r10,rbp + adc r11,rdx + adc r12,0 + + + + + mov rdi,QWORD PTR[16+rsp] + sub r14,QWORD PTR[rcx] + mov rdx,r15 + sbb r15,QWORD PTR[8+rcx] + mov rbx,r8 + sbb r8,QWORD PTR[16+rcx] + mov rsi,r9 + sbb r9,QWORD PTR[24+rcx] + mov rbp,r10 + sbb r10,QWORD PTR[32+rcx] + mov r13,r11 + sbb r11,QWORD PTR[40+rcx] + sbb r12,0 + + cmovc r14,rax + cmovc r15,rdx + cmovc r8,rbx + mov QWORD PTR[rdi],r14 + cmovc r9,rsi + mov QWORD PTR[8+rdi],r15 + cmovc r10,rbp + mov QWORD PTR[16+rdi],r8 + cmovc r11,r13 + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + DB 0F3h,0C3h ;repret +__mulq_mont_384 ENDP +PUBLIC sqr_n_mul_mont_384 + + +ALIGN 32 +sqr_n_mul_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_n_mul_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + mov r9,QWORD PTR[48+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8*17 + +$L$SEH_body_sqr_n_mul_mont_384:: + + + mov QWORD PTR[rsp],r8 + mov QWORD PTR[8+rsp],rdi + mov QWORD PTR[16+rsp],rcx + lea rdi,QWORD PTR[32+rsp] + mov QWORD PTR[24+rsp],r9 + movq xmm2,QWORD PTR[r9] + +$L$oop_sqr_384:: + movd xmm1,edx + + call __sqrq_384 + + lea rsi,QWORD PTR[rdi] + mov rcx,QWORD PTR[rsp] + mov rbx,QWORD PTR[16+rsp] + call __mulq_by_1_mont_384 + call __redc_tail_mont_384 + + movd edx,xmm1 + lea rsi,QWORD PTR[rdi] + dec edx + jnz $L$oop_sqr_384 + +DB 102,72,15,126,208 + mov rcx,rbx + mov rbx,QWORD PTR[24+rsp] + + + + + + + mov r12,r8 + mov r13,r9 + + call __mulq_mont_384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[136+rsp] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqr_n_mul_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_n_mul_mont_384:: +sqr_n_mul_mont_384 ENDP + +PUBLIC sqr_n_mul_mont_383 + + +ALIGN 32 +sqr_n_mul_mont_383 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_n_mul_mont_383:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + mov r9,QWORD PTR[48+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8*17 + +$L$SEH_body_sqr_n_mul_mont_383:: + + + mov QWORD PTR[rsp],r8 + mov QWORD PTR[8+rsp],rdi + mov QWORD PTR[16+rsp],rcx + lea rdi,QWORD PTR[32+rsp] + mov QWORD PTR[24+rsp],r9 + movq xmm2,QWORD PTR[r9] + +$L$oop_sqr_383:: + movd xmm1,edx + + call __sqrq_384 + + lea rsi,QWORD PTR[rdi] + mov rcx,QWORD PTR[rsp] + mov rbx,QWORD PTR[16+rsp] + call __mulq_by_1_mont_384 + + movd edx,xmm1 + add r14,QWORD PTR[48+rsi] + adc r15,QWORD PTR[56+rsi] + adc r8,QWORD PTR[64+rsi] + adc r9,QWORD PTR[72+rsi] + adc r10,QWORD PTR[80+rsi] + adc r11,QWORD PTR[88+rsi] + lea rsi,QWORD PTR[rdi] + + mov QWORD PTR[rdi],r14 + mov QWORD PTR[8+rdi],r15 + mov QWORD PTR[16+rdi],r8 + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + dec edx + jnz $L$oop_sqr_383 + +DB 102,72,15,126,208 + mov rcx,rbx + mov rbx,QWORD PTR[24+rsp] + + + + + + + mov r12,r8 + mov r13,r9 + + call __mulq_mont_384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[136+rsp] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqr_n_mul_mont_383:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_n_mul_mont_383:: +sqr_n_mul_mont_383 ENDP + +ALIGN 32 +__mulq_mont_383_nonred PROC PRIVATE + DB 243,15,30,250 + mov rbp,rax + mul r14 + mov r8,rax + mov rax,rbp + mov r9,rdx + + mul r15 + add r9,rax + mov rax,rbp + adc rdx,0 + mov r10,rdx + + mul r12 + add r10,rax + mov rax,rbp + adc rdx,0 + mov r11,rdx + + mov r15,r8 + imul r8,QWORD PTR[8+rsp] + + mul r13 + add r11,rax + mov rax,rbp + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[32+rsi] + add r12,rax + mov rax,rbp + adc rdx,0 + mov r13,rdx + + mul QWORD PTR[40+rsi] + add r13,rax + mov rax,r8 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[rcx] + add r15,rax + mov rax,r8 + adc r15,rdx + + mul QWORD PTR[8+rcx] + add r9,rax + mov rax,r8 + adc rdx,0 + add r9,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[16+rcx] + add r10,rax + mov rax,r8 + adc rdx,0 + add r10,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[24+rcx] + add r11,r15 + adc rdx,0 + add r11,rax + mov rax,r8 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[32+rcx] + add r12,rax + mov rax,r8 + adc rdx,0 + add r12,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[40+rcx] + add r13,rax + mov rax,QWORD PTR[8+rbx] + adc rdx,0 + add r13,r15 + adc r14,rdx + + mov rbp,rax + mul QWORD PTR[rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[8+rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + add r10,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[16+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r11,r15 + adc rdx,0 + mov r15,rdx + + mov r8,r9 + imul r9,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r12,rax + mov rax,rbp + adc rdx,0 + add r12,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[32+rsi] + add r13,rax + mov rax,rbp + adc rdx,0 + add r13,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[40+rsi] + add r14,r15 + adc rdx,0 + add r14,rax + mov rax,r9 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[rcx] + add r8,rax + mov rax,r9 + adc r8,rdx + + mul QWORD PTR[8+rcx] + add r10,rax + mov rax,r9 + adc rdx,0 + add r10,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rcx] + add r11,rax + mov rax,r9 + adc rdx,0 + add r11,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[24+rcx] + add r12,r8 + adc rdx,0 + add r12,rax + mov rax,r9 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[32+rcx] + add r13,rax + mov rax,r9 + adc rdx,0 + add r13,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[40+rcx] + add r14,rax + mov rax,QWORD PTR[16+rbx] + adc rdx,0 + add r14,r8 + adc r15,rdx + + mov rbp,rax + mul QWORD PTR[rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[8+rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + add r11,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[16+rsi] + add r12,rax + mov rax,rbp + adc rdx,0 + add r12,r8 + adc rdx,0 + mov r8,rdx + + mov r9,r10 + imul r10,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r13,rax + mov rax,rbp + adc rdx,0 + add r13,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[32+rsi] + add r14,rax + mov rax,rbp + adc rdx,0 + add r14,r8 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[40+rsi] + add r15,r8 + adc rdx,0 + add r15,rax + mov rax,r10 + adc rdx,0 + mov r8,rdx + + mul QWORD PTR[rcx] + add r9,rax + mov rax,r10 + adc r9,rdx + + mul QWORD PTR[8+rcx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[16+rcx] + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[24+rcx] + add r13,r9 + adc rdx,0 + add r13,rax + mov rax,r10 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[32+rcx] + add r14,rax + mov rax,r10 + adc rdx,0 + add r14,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[40+rcx] + add r15,rax + mov rax,QWORD PTR[24+rbx] + adc rdx,0 + add r15,r9 + adc r8,rdx + + mov rbp,rax + mul QWORD PTR[rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[8+rsi] + add r12,rax + mov rax,rbp + adc rdx,0 + add r12,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[16+rsi] + add r13,rax + mov rax,rbp + adc rdx,0 + add r13,r9 + adc rdx,0 + mov r9,rdx + + mov r10,r11 + imul r11,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r14,rax + mov rax,rbp + adc rdx,0 + add r14,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[32+rsi] + add r15,rax + mov rax,rbp + adc rdx,0 + add r15,r9 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[40+rsi] + add r8,r9 + adc rdx,0 + add r8,rax + mov rax,r11 + adc rdx,0 + mov r9,rdx + + mul QWORD PTR[rcx] + add r10,rax + mov rax,r11 + adc r10,rdx + + mul QWORD PTR[8+rcx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[16+rcx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[24+rcx] + add r14,r10 + adc rdx,0 + add r14,rax + mov rax,r11 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rcx] + add r15,rax + mov rax,r11 + adc rdx,0 + add r15,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[40+rcx] + add r8,rax + mov rax,QWORD PTR[32+rbx] + adc rdx,0 + add r8,r10 + adc r9,rdx + + mov rbp,rax + mul QWORD PTR[rsi] + add r12,rax + mov rax,rbp + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[8+rsi] + add r13,rax + mov rax,rbp + adc rdx,0 + add r13,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[16+rsi] + add r14,rax + mov rax,rbp + adc rdx,0 + add r14,r10 + adc rdx,0 + mov r10,rdx + + mov r11,r12 + imul r12,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r15,rax + mov rax,rbp + adc rdx,0 + add r15,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[32+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add r8,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[40+rsi] + add r9,r10 + adc rdx,0 + add r9,rax + mov rax,r12 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[rcx] + add r11,rax + mov rax,r12 + adc r11,rdx + + mul QWORD PTR[8+rcx] + add r13,rax + mov rax,r12 + adc rdx,0 + add r13,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[16+rcx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[24+rcx] + add r15,r11 + adc rdx,0 + add r15,rax + mov rax,r12 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[32+rcx] + add r8,rax + mov rax,r12 + adc rdx,0 + add r8,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rcx] + add r9,rax + mov rax,QWORD PTR[40+rbx] + adc rdx,0 + add r9,r11 + adc r10,rdx + + mov rbp,rax + mul QWORD PTR[rsi] + add r13,rax + mov rax,rbp + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[8+rsi] + add r14,rax + mov rax,rbp + adc rdx,0 + add r14,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[16+rsi] + add r15,rax + mov rax,rbp + adc rdx,0 + add r15,r11 + adc rdx,0 + mov r11,rdx + + mov r12,r13 + imul r13,QWORD PTR[8+rsp] + + mul QWORD PTR[24+rsi] + add r8,rax + mov rax,rbp + adc rdx,0 + add r8,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[32+rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + add r9,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[40+rsi] + add r10,r11 + adc rdx,0 + add r10,rax + mov rax,r13 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[rcx] + add r12,rax + mov rax,r13 + adc r12,rdx + + mul QWORD PTR[8+rcx] + add r14,rax + mov rax,r13 + adc rdx,0 + add r14,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[16+rcx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[24+rcx] + add r8,r12 + adc rdx,0 + add r8,rax + mov rax,r13 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[32+rcx] + add r9,rax + mov rax,r13 + adc rdx,0 + add r9,r12 + adc rdx,0 + mov r12,rdx + + mul QWORD PTR[40+rcx] + add r10,rax + mov rax,r14 + adc rdx,0 + add r10,r12 + adc r11,rdx + DB 0F3h,0C3h ;repret +__mulq_mont_383_nonred ENDP +PUBLIC sqr_mont_382x + + +ALIGN 32 +sqr_mont_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqr_mont_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_sqr_mont_382x:: + + + mov QWORD PTR[rsp],rcx + mov rcx,rdx + mov QWORD PTR[16+rsp],rsi + mov QWORD PTR[24+rsp],rdi + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,r8 + add r8,QWORD PTR[48+rsi] + mov r15,r9 + adc r9,QWORD PTR[56+rsi] + mov rax,r10 + adc r10,QWORD PTR[64+rsi] + mov rdx,r11 + adc r11,QWORD PTR[72+rsi] + mov rbx,r12 + adc r12,QWORD PTR[80+rsi] + mov rbp,r13 + adc r13,QWORD PTR[88+rsi] + + sub r14,QWORD PTR[48+rsi] + sbb r15,QWORD PTR[56+rsi] + sbb rax,QWORD PTR[64+rsi] + sbb rdx,QWORD PTR[72+rsi] + sbb rbx,QWORD PTR[80+rsi] + sbb rbp,QWORD PTR[88+rsi] + sbb rdi,rdi + + mov QWORD PTR[((32+0))+rsp],r8 + mov QWORD PTR[((32+8))+rsp],r9 + mov QWORD PTR[((32+16))+rsp],r10 + mov QWORD PTR[((32+24))+rsp],r11 + mov QWORD PTR[((32+32))+rsp],r12 + mov QWORD PTR[((32+40))+rsp],r13 + + mov QWORD PTR[((32+48))+rsp],r14 + mov QWORD PTR[((32+56))+rsp],r15 + mov QWORD PTR[((32+64))+rsp],rax + mov QWORD PTR[((32+72))+rsp],rdx + mov QWORD PTR[((32+80))+rsp],rbx + mov QWORD PTR[((32+88))+rsp],rbp + mov QWORD PTR[((32+96))+rsp],rdi + + + + lea rbx,QWORD PTR[48+rsi] + + mov rax,QWORD PTR[48+rsi] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov r12,QWORD PTR[16+rsi] + mov r13,QWORD PTR[24+rsi] + + mov rdi,QWORD PTR[24+rsp] + call __mulq_mont_383_nonred + add r14,r14 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],r8 + mov QWORD PTR[72+rdi],r9 + mov QWORD PTR[80+rdi],r10 + mov QWORD PTR[88+rdi],r11 + + lea rsi,QWORD PTR[32+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + + mov rax,QWORD PTR[((32+48))+rsp] + mov r14,QWORD PTR[((32+0))+rsp] + mov r15,QWORD PTR[((32+8))+rsp] + mov r12,QWORD PTR[((32+16))+rsp] + mov r13,QWORD PTR[((32+24))+rsp] + + call __mulq_mont_383_nonred + mov rsi,QWORD PTR[((32+96))+rsp] + mov r12,QWORD PTR[((32+0))+rsp] + mov r13,QWORD PTR[((32+8))+rsp] + and r12,rsi + mov rax,QWORD PTR[((32+16))+rsp] + and r13,rsi + mov rbx,QWORD PTR[((32+24))+rsp] + and rax,rsi + mov rbp,QWORD PTR[((32+32))+rsp] + and rbx,rsi + and rbp,rsi + and rsi,QWORD PTR[((32+40))+rsp] + + sub r14,r12 + mov r12,QWORD PTR[rcx] + sbb r15,r13 + mov r13,QWORD PTR[8+rcx] + sbb r8,rax + mov rax,QWORD PTR[16+rcx] + sbb r9,rbx + mov rbx,QWORD PTR[24+rcx] + sbb r10,rbp + mov rbp,QWORD PTR[32+rcx] + sbb r11,rsi + sbb rsi,rsi + + and r12,rsi + and r13,rsi + and rax,rsi + and rbx,rsi + and rbp,rsi + and rsi,QWORD PTR[40+rcx] + + add r14,r12 + adc r15,r13 + adc r8,rax + adc r9,rbx + adc r10,rbp + adc r11,rsi + + mov QWORD PTR[rdi],r14 + mov QWORD PTR[8+rdi],r15 + mov QWORD PTR[16+rdi],r8 + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqr_mont_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqr_mont_382x:: +sqr_mont_382x ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mul_mont_384x + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_prologue + + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_body + + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_end_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_epilogue + + DD imagerel $L$SEH_begin_sqr_mont_384x + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_prologue + + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_body + + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_end_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_epilogue + + DD imagerel $L$SEH_begin_mul_382x + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_info_mul_382x_prologue + + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_info_mul_382x_body + + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_end_mul_382x + DD imagerel $L$SEH_info_mul_382x_epilogue + + DD imagerel $L$SEH_begin_sqr_382x + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_prologue + + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_body + + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_end_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_epilogue + + DD imagerel $L$SEH_begin_mul_384 + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_info_mul_384_prologue + + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_info_mul_384_body + + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_end_mul_384 + DD imagerel $L$SEH_info_mul_384_epilogue + + DD imagerel $L$SEH_begin_sqr_384 + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_info_sqr_384_prologue + + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_info_sqr_384_body + + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_end_sqr_384 + DD imagerel $L$SEH_info_sqr_384_epilogue + + DD imagerel $L$SEH_begin_sqr_mont_384 + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_prologue + + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_body + + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_end_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_epilogue + + DD imagerel $L$SEH_begin_redc_mont_384 + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_prologue + + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_body + + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_end_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_epilogue + + DD imagerel $L$SEH_begin_from_mont_384 + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_prologue + + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_body + + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_end_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_epilogue + + DD imagerel $L$SEH_begin_sgn0_pty_mont_384 + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_prologue + + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_body + + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 + DD imagerel $L$SEH_end_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_epilogue + + DD imagerel $L$SEH_begin_sgn0_pty_mont_384x + DD imagerel $L$SEH_body_sgn0_pty_mont_384x + DD imagerel $L$SEH_info_sgn0_pty_mont_384x_prologue + + DD imagerel $L$SEH_body_sgn0_pty_mont_384x + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384x + DD imagerel $L$SEH_info_sgn0_pty_mont_384x_body + + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384x + DD imagerel $L$SEH_end_sgn0_pty_mont_384x + DD imagerel $L$SEH_info_sgn0_pty_mont_384x_epilogue + + DD imagerel $L$SEH_begin_mul_mont_384 + DD imagerel $L$SEH_body_mul_mont_384 + DD imagerel $L$SEH_info_mul_mont_384_prologue + + DD imagerel $L$SEH_body_mul_mont_384 + DD imagerel $L$SEH_epilogue_mul_mont_384 + DD imagerel $L$SEH_info_mul_mont_384_body + + DD imagerel $L$SEH_epilogue_mul_mont_384 + DD imagerel $L$SEH_end_mul_mont_384 + DD imagerel $L$SEH_info_mul_mont_384_epilogue + + DD imagerel $L$SEH_begin_sqr_n_mul_mont_384 + DD imagerel $L$SEH_body_sqr_n_mul_mont_384 + DD imagerel $L$SEH_info_sqr_n_mul_mont_384_prologue + + DD imagerel $L$SEH_body_sqr_n_mul_mont_384 + DD imagerel $L$SEH_epilogue_sqr_n_mul_mont_384 + DD imagerel $L$SEH_info_sqr_n_mul_mont_384_body + + DD imagerel $L$SEH_epilogue_sqr_n_mul_mont_384 + DD imagerel $L$SEH_end_sqr_n_mul_mont_384 + DD imagerel $L$SEH_info_sqr_n_mul_mont_384_epilogue + + DD imagerel $L$SEH_begin_sqr_n_mul_mont_383 + DD imagerel $L$SEH_body_sqr_n_mul_mont_383 + DD imagerel $L$SEH_info_sqr_n_mul_mont_383_prologue + + DD imagerel $L$SEH_body_sqr_n_mul_mont_383 + DD imagerel $L$SEH_epilogue_sqr_n_mul_mont_383 + DD imagerel $L$SEH_info_sqr_n_mul_mont_383_body + + DD imagerel $L$SEH_epilogue_sqr_n_mul_mont_383 + DD imagerel $L$SEH_end_sqr_n_mul_mont_383 + DD imagerel $L$SEH_info_sqr_n_mul_mont_383_epilogue + + DD imagerel $L$SEH_begin_sqr_mont_382x + DD imagerel $L$SEH_body_sqr_mont_382x + DD imagerel $L$SEH_info_sqr_mont_382x_prologue + + DD imagerel $L$SEH_body_sqr_mont_382x + DD imagerel $L$SEH_epilogue_sqr_mont_382x + DD imagerel $L$SEH_info_sqr_mont_382x_body + + DD imagerel $L$SEH_epilogue_sqr_mont_382x + DD imagerel $L$SEH_end_sqr_mont_382x + DD imagerel $L$SEH_info_sqr_mont_382x_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_mul_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_mont_384x_body:: +DB 1,0,18,0 +DB 000h,0f4h,029h,000h +DB 000h,0e4h,02ah,000h +DB 000h,0d4h,02bh,000h +DB 000h,0c4h,02ch,000h +DB 000h,034h,02dh,000h +DB 000h,054h,02eh,000h +DB 000h,074h,030h,000h +DB 000h,064h,031h,000h +DB 000h,001h,02fh,000h +$L$SEH_info_mul_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_mont_384x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqr_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_382x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_mul_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_382x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqr_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_384_body:: +DB 1,0,11,0 +DB 000h,0c4h,000h,000h +DB 000h,034h,001h,000h +DB 000h,054h,002h,000h +DB 000h,074h,004h,000h +DB 000h,064h,005h,000h +DB 000h,022h +DB 000h,000h,000h,000h,000h,000h +$L$SEH_info_mul_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqr_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_mont_384_body:: +DB 1,0,18,0 +DB 000h,0f4h,00fh,000h +DB 000h,0e4h,010h,000h +DB 000h,0d4h,011h,000h +DB 000h,0c4h,012h,000h +DB 000h,034h,013h,000h +DB 000h,054h,014h,000h +DB 000h,074h,016h,000h +DB 000h,064h,017h,000h +DB 000h,001h,015h,000h +$L$SEH_info_sqr_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_redc_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_redc_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_redc_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_from_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_from_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_from_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0_pty_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0_pty_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sgn0_pty_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0_pty_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0_pty_mont_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sgn0_pty_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mul_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mul_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,003h,000h +DB 000h,0e4h,004h,000h +DB 000h,0d4h,005h,000h +DB 000h,0c4h,006h,000h +DB 000h,034h,007h,000h +DB 000h,054h,008h,000h +DB 000h,074h,00ah,000h +DB 000h,064h,00bh,000h +DB 000h,082h +DB 000h,000h +$L$SEH_info_mul_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_n_mul_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_n_mul_mont_384_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqr_n_mul_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_n_mul_mont_383_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_n_mul_mont_383_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqr_n_mul_mont_383_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqr_mont_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqr_mont_382x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqr_mont_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/mulx_mont_256-x86_64.asm b/crypto/blst_src/build/win64/mulx_mont_256-x86_64.asm new file mode 100644 index 00000000000..83534c629e9 --- /dev/null +++ b/crypto/blst_src/build/win64/mulx_mont_256-x86_64.asm @@ -0,0 +1,796 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +PUBLIC mulx_mont_sparse_256 + + +ALIGN 32 +mulx_mont_sparse_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mulx_mont_sparse_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_mulx_mont_sparse_256:: + + + mov rbx,rdx + mov rdx,QWORD PTR[rdx] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rbp,QWORD PTR[16+rsi] + mov r9,QWORD PTR[24+rsi] + lea rsi,QWORD PTR[((-128))+rsi] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r11,rax,r14 + call __mulx_mont_sparse_256 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_mulx_mont_sparse_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mulx_mont_sparse_256:: +mulx_mont_sparse_256 ENDP + +PUBLIC sqrx_mont_sparse_256 + + +ALIGN 32 +sqrx_mont_sparse_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_mont_sparse_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sqrx_mont_sparse_256:: + + + mov rbx,rsi + mov r8,rcx + mov rcx,rdx + mov rdx,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rbp,QWORD PTR[16+rsi] + mov r9,QWORD PTR[24+rsi] + lea rsi,QWORD PTR[((-128))+rbx] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r11,rax,rdx + call __mulx_mont_sparse_256 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqrx_mont_sparse_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_mont_sparse_256:: +sqrx_mont_sparse_256 ENDP + +ALIGN 32 +__mulx_mont_sparse_256 PROC PRIVATE + DB 243,15,30,250 + mulx r12,r15,r15 + mulx r13,rbp,rbp + add r11,r15 + mulx r14,r9,r9 + mov rdx,QWORD PTR[8+rbx] + adc r12,rbp + adc r13,r9 + adc r14,0 + + mov r10,rax + imul rax,r8 + + + xor r15,r15 + mulx r9,rbp,QWORD PTR[((0+128))+rsi] + adox r11,rbp + adcx r12,r9 + + mulx r9,rbp,QWORD PTR[((8+128))+rsi] + adox r12,rbp + adcx r13,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rsi] + adox r13,rbp + adcx r14,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rsi] + mov rdx,rax + adox r14,rbp + adcx r9,r15 + adox r15,r9 + + + mulx rax,rbp,QWORD PTR[((0+128))+rcx] + adcx r10,rbp + adox rax,r11 + + mulx r9,rbp,QWORD PTR[((8+128))+rcx] + adcx rax,rbp + adox r12,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rcx] + adcx r12,rbp + adox r13,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rcx] + mov rdx,QWORD PTR[16+rbx] + adcx r13,rbp + adox r14,r9 + adcx r14,r10 + adox r15,r10 + adcx r15,r10 + adox r10,r10 + adc r10,0 + mov r11,rax + imul rax,r8 + + + xor rbp,rbp + mulx r9,rbp,QWORD PTR[((0+128))+rsi] + adox r12,rbp + adcx r13,r9 + + mulx r9,rbp,QWORD PTR[((8+128))+rsi] + adox r13,rbp + adcx r14,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rsi] + adox r14,rbp + adcx r15,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rsi] + mov rdx,rax + adox r15,rbp + adcx r9,r10 + adox r10,r9 + + + mulx rax,rbp,QWORD PTR[((0+128))+rcx] + adcx r11,rbp + adox rax,r12 + + mulx r9,rbp,QWORD PTR[((8+128))+rcx] + adcx rax,rbp + adox r13,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rcx] + adcx r13,rbp + adox r14,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rcx] + mov rdx,QWORD PTR[24+rbx] + adcx r14,rbp + adox r15,r9 + adcx r15,r11 + adox r10,r11 + adcx r10,r11 + adox r11,r11 + adc r11,0 + mov r12,rax + imul rax,r8 + + + xor rbp,rbp + mulx r9,rbp,QWORD PTR[((0+128))+rsi] + adox r13,rbp + adcx r14,r9 + + mulx r9,rbp,QWORD PTR[((8+128))+rsi] + adox r14,rbp + adcx r15,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rsi] + adox r15,rbp + adcx r10,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rsi] + mov rdx,rax + adox r10,rbp + adcx r9,r11 + adox r11,r9 + + + mulx rax,rbp,QWORD PTR[((0+128))+rcx] + adcx r12,rbp + adox rax,r13 + + mulx r9,rbp,QWORD PTR[((8+128))+rcx] + adcx rax,rbp + adox r14,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rcx] + adcx r14,rbp + adox r15,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rcx] + mov rdx,rax + adcx r15,rbp + adox r10,r9 + adcx r10,r12 + adox r11,r12 + adcx r11,r12 + adox r12,r12 + adc r12,0 + imul rdx,r8 + + + xor rbp,rbp + mulx r9,r13,QWORD PTR[((0+128))+rcx] + adcx r13,rax + adox r14,r9 + + mulx r9,rbp,QWORD PTR[((8+128))+rcx] + adcx r14,rbp + adox r15,r9 + + mulx r9,rbp,QWORD PTR[((16+128))+rcx] + adcx r15,rbp + adox r10,r9 + + mulx r9,rbp,QWORD PTR[((24+128))+rcx] + mov rdx,r14 + lea rcx,QWORD PTR[128+rcx] + adcx r10,rbp + adox r11,r9 + mov rax,r15 + adcx r11,r13 + adox r12,r13 + adc r12,0 + + + + + mov rbp,r10 + sub r14,QWORD PTR[rcx] + sbb r15,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rcx] + mov r9,r11 + sbb r11,QWORD PTR[24+rcx] + sbb r12,0 + + cmovc r14,rdx + cmovc r15,rax + cmovc r10,rbp + mov QWORD PTR[rdi],r14 + cmovc r11,r9 + mov QWORD PTR[8+rdi],r15 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + + DB 0F3h,0C3h ;repret +__mulx_mont_sparse_256 ENDP +PUBLIC fromx_mont_256 + + +ALIGN 32 +fromx_mont_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_fromx_mont_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_fromx_mont_256:: + + + mov rbx,rdx + call __mulx_by_1_mont_256 + + + + + + mov rdx,r15 + mov r12,r10 + mov r13,r11 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r10,QWORD PTR[16+rbx] + sbb r11,QWORD PTR[24+rbx] + + cmovnc rax,r14 + cmovnc rdx,r15 + cmovnc r12,r10 + mov QWORD PTR[rdi],rax + cmovnc r13,r11 + mov QWORD PTR[8+rdi],rdx + mov QWORD PTR[16+rdi],r12 + mov QWORD PTR[24+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_fromx_mont_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_fromx_mont_256:: +fromx_mont_256 ENDP + +PUBLIC redcx_mont_256 + + +ALIGN 32 +redcx_mont_256 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_redcx_mont_256:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_redcx_mont_256:: + + + mov rbx,rdx + call __mulx_by_1_mont_256 + + add r14,QWORD PTR[32+rsi] + adc r15,QWORD PTR[40+rsi] + mov rax,r14 + adc r10,QWORD PTR[48+rsi] + mov rdx,r15 + adc r11,QWORD PTR[56+rsi] + sbb rsi,rsi + + + + + mov r12,r10 + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r10,QWORD PTR[16+rbx] + mov r13,r11 + sbb r11,QWORD PTR[24+rbx] + sbb rsi,0 + + cmovnc rax,r14 + cmovnc rdx,r15 + cmovnc r12,r10 + mov QWORD PTR[rdi],rax + cmovnc r13,r11 + mov QWORD PTR[8+rdi],rdx + mov QWORD PTR[16+rdi],r12 + mov QWORD PTR[24+rdi],r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_redcx_mont_256:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_redcx_mont_256:: +redcx_mont_256 ENDP + +ALIGN 32 +__mulx_by_1_mont_256 PROC PRIVATE + DB 243,15,30,250 + mov rax,QWORD PTR[rsi] + mov r11,QWORD PTR[8+rsi] + mov r12,QWORD PTR[16+rsi] + mov r13,QWORD PTR[24+rsi] + + mov r14,rax + imul rax,rcx + mov r10,rax + + mul QWORD PTR[rbx] + add r14,rax + mov rax,r10 + adc r14,rdx + + mul QWORD PTR[8+rbx] + add r11,rax + mov rax,r10 + adc rdx,0 + add r11,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[16+rbx] + mov r15,r11 + imul r11,rcx + add r12,rax + mov rax,r10 + adc rdx,0 + add r12,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[24+rbx] + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r14 + adc rdx,0 + mov r14,rdx + + mul QWORD PTR[rbx] + add r15,rax + mov rax,r11 + adc r15,rdx + + mul QWORD PTR[8+rbx] + add r12,rax + mov rax,r11 + adc rdx,0 + add r12,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[16+rbx] + mov r10,r12 + imul r12,rcx + add r13,rax + mov rax,r11 + adc rdx,0 + add r13,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[24+rbx] + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r15 + adc rdx,0 + mov r15,rdx + + mul QWORD PTR[rbx] + add r10,rax + mov rax,r12 + adc r10,rdx + + mul QWORD PTR[8+rbx] + add r13,rax + mov rax,r12 + adc rdx,0 + add r13,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[16+rbx] + mov r11,r13 + imul r13,rcx + add r14,rax + mov rax,r12 + adc rdx,0 + add r14,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[24+rbx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,r10 + adc rdx,0 + mov r10,rdx + + mul QWORD PTR[rbx] + add r11,rax + mov rax,r13 + adc r11,rdx + + mul QWORD PTR[8+rbx] + add r14,rax + mov rax,r13 + adc rdx,0 + add r14,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[16+rbx] + add r15,rax + mov rax,r13 + adc rdx,0 + add r15,r11 + adc rdx,0 + mov r11,rdx + + mul QWORD PTR[24+rbx] + add r10,rax + mov rax,r14 + adc rdx,0 + add r10,r11 + adc rdx,0 + mov r11,rdx + DB 0F3h,0C3h ;repret +__mulx_by_1_mont_256 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mulx_mont_sparse_256 + DD imagerel $L$SEH_body_mulx_mont_sparse_256 + DD imagerel $L$SEH_info_mulx_mont_sparse_256_prologue + + DD imagerel $L$SEH_body_mulx_mont_sparse_256 + DD imagerel $L$SEH_epilogue_mulx_mont_sparse_256 + DD imagerel $L$SEH_info_mulx_mont_sparse_256_body + + DD imagerel $L$SEH_epilogue_mulx_mont_sparse_256 + DD imagerel $L$SEH_end_mulx_mont_sparse_256 + DD imagerel $L$SEH_info_mulx_mont_sparse_256_epilogue + + DD imagerel $L$SEH_begin_sqrx_mont_sparse_256 + DD imagerel $L$SEH_body_sqrx_mont_sparse_256 + DD imagerel $L$SEH_info_sqrx_mont_sparse_256_prologue + + DD imagerel $L$SEH_body_sqrx_mont_sparse_256 + DD imagerel $L$SEH_epilogue_sqrx_mont_sparse_256 + DD imagerel $L$SEH_info_sqrx_mont_sparse_256_body + + DD imagerel $L$SEH_epilogue_sqrx_mont_sparse_256 + DD imagerel $L$SEH_end_sqrx_mont_sparse_256 + DD imagerel $L$SEH_info_sqrx_mont_sparse_256_epilogue + + DD imagerel $L$SEH_begin_fromx_mont_256 + DD imagerel $L$SEH_body_fromx_mont_256 + DD imagerel $L$SEH_info_fromx_mont_256_prologue + + DD imagerel $L$SEH_body_fromx_mont_256 + DD imagerel $L$SEH_epilogue_fromx_mont_256 + DD imagerel $L$SEH_info_fromx_mont_256_body + + DD imagerel $L$SEH_epilogue_fromx_mont_256 + DD imagerel $L$SEH_end_fromx_mont_256 + DD imagerel $L$SEH_info_fromx_mont_256_epilogue + + DD imagerel $L$SEH_begin_redcx_mont_256 + DD imagerel $L$SEH_body_redcx_mont_256 + DD imagerel $L$SEH_info_redcx_mont_256_prologue + + DD imagerel $L$SEH_body_redcx_mont_256 + DD imagerel $L$SEH_epilogue_redcx_mont_256 + DD imagerel $L$SEH_info_redcx_mont_256_body + + DD imagerel $L$SEH_epilogue_redcx_mont_256 + DD imagerel $L$SEH_end_redcx_mont_256 + DD imagerel $L$SEH_info_redcx_mont_256_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_mulx_mont_sparse_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mulx_mont_sparse_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_mulx_mont_sparse_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_mont_sparse_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_mont_sparse_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqrx_mont_sparse_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_fromx_mont_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_fromx_mont_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_fromx_mont_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_redcx_mont_256_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_redcx_mont_256_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_redcx_mont_256_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/mulx_mont_384-x86_64.asm b/crypto/blst_src/build/win64/mulx_mont_384-x86_64.asm new file mode 100644 index 00000000000..25bee97731b --- /dev/null +++ b/crypto/blst_src/build/win64/mulx_mont_384-x86_64.asm @@ -0,0 +1,3586 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + + + + + + + + +ALIGN 32 +__sub_mod_384x384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] + + sub r8,QWORD PTR[rdx] + mov r15,QWORD PTR[56+rsi] + sbb r9,QWORD PTR[8+rdx] + mov rax,QWORD PTR[64+rsi] + sbb r10,QWORD PTR[16+rdx] + mov rbx,QWORD PTR[72+rsi] + sbb r11,QWORD PTR[24+rdx] + mov rbp,QWORD PTR[80+rsi] + sbb r12,QWORD PTR[32+rdx] + mov rsi,QWORD PTR[88+rsi] + sbb r13,QWORD PTR[40+rdx] + mov QWORD PTR[rdi],r8 + sbb r14,QWORD PTR[48+rdx] + mov r8,QWORD PTR[rcx] + mov QWORD PTR[8+rdi],r9 + sbb r15,QWORD PTR[56+rdx] + mov r9,QWORD PTR[8+rcx] + mov QWORD PTR[16+rdi],r10 + sbb rax,QWORD PTR[64+rdx] + mov r10,QWORD PTR[16+rcx] + mov QWORD PTR[24+rdi],r11 + sbb rbx,QWORD PTR[72+rdx] + mov r11,QWORD PTR[24+rcx] + mov QWORD PTR[32+rdi],r12 + sbb rbp,QWORD PTR[80+rdx] + mov r12,QWORD PTR[32+rcx] + mov QWORD PTR[40+rdi],r13 + sbb rsi,QWORD PTR[88+rdx] + mov r13,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r8,rdx + and r9,rdx + and r10,rdx + and r11,rdx + and r12,rdx + and r13,rdx + + add r14,r8 + adc r15,r9 + mov QWORD PTR[48+rdi],r14 + adc rax,r10 + mov QWORD PTR[56+rdi],r15 + adc rbx,r11 + mov QWORD PTR[64+rdi],rax + adc rbp,r12 + mov QWORD PTR[72+rdi],rbx + adc rsi,r13 + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rsi + + DB 0F3h,0C3h ;repret +__sub_mod_384x384 ENDP + + +ALIGN 32 +__add_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + add r8,QWORD PTR[rdx] + adc r9,QWORD PTR[8+rdx] + adc r10,QWORD PTR[16+rdx] + mov r14,r8 + adc r11,QWORD PTR[24+rdx] + mov r15,r9 + adc r12,QWORD PTR[32+rdx] + mov rax,r10 + adc r13,QWORD PTR[40+rdx] + mov rbx,r11 + sbb rdx,rdx + + sub r8,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rcx] + mov rbp,r12 + sbb r10,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rcx] + mov rsi,r13 + sbb r13,QWORD PTR[40+rcx] + sbb rdx,0 + + cmovc r8,r14 + cmovc r9,r15 + cmovc r10,rax + mov QWORD PTR[rdi],r8 + cmovc r11,rbx + mov QWORD PTR[8+rdi],r9 + cmovc r12,rbp + mov QWORD PTR[16+rdi],r10 + cmovc r13,rsi + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__add_mod_384 ENDP + + +ALIGN 32 +__sub_mod_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + +__sub_mod_384_a_is_loaded:: + sub r8,QWORD PTR[rdx] + mov r14,QWORD PTR[rcx] + sbb r9,QWORD PTR[8+rdx] + mov r15,QWORD PTR[8+rcx] + sbb r10,QWORD PTR[16+rdx] + mov rax,QWORD PTR[16+rcx] + sbb r11,QWORD PTR[24+rdx] + mov rbx,QWORD PTR[24+rcx] + sbb r12,QWORD PTR[32+rdx] + mov rbp,QWORD PTR[32+rcx] + sbb r13,QWORD PTR[40+rdx] + mov rsi,QWORD PTR[40+rcx] + sbb rdx,rdx + + and r14,rdx + and r15,rdx + and rax,rdx + and rbx,rdx + and rbp,rdx + and rsi,rdx + + add r8,r14 + adc r9,r15 + mov QWORD PTR[rdi],r8 + adc r10,rax + mov QWORD PTR[8+rdi],r9 + adc r11,rbx + mov QWORD PTR[16+rdi],r10 + adc r12,rbp + mov QWORD PTR[24+rdi],r11 + adc r13,rsi + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + + DB 0F3h,0C3h ;repret +__sub_mod_384 ENDP +PUBLIC mulx_mont_384x + + +ALIGN 32 +mulx_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mulx_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,328 + +$L$SEH_body_mulx_mont_384x:: + + + mov rbx,rdx + mov QWORD PTR[32+rsp],rdi + mov QWORD PTR[24+rsp],rsi + mov QWORD PTR[16+rsp],rdx + mov QWORD PTR[8+rsp],rcx + mov QWORD PTR[rsp],r8 + + + + + lea rdi,QWORD PTR[40+rsp] + call __mulx_384 + + + lea rbx,QWORD PTR[48+rbx] + lea rsi,QWORD PTR[((128+48))+rsi] + lea rdi,QWORD PTR[96+rdi] + call __mulx_384 + + + mov rcx,QWORD PTR[8+rsp] + lea rsi,QWORD PTR[rbx] + lea rdx,QWORD PTR[((-48))+rbx] + lea rdi,QWORD PTR[((40+192+48))+rsp] + call __add_mod_384 + + mov rsi,QWORD PTR[24+rsp] + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[((-48))+rdi] + call __add_mod_384 + + lea rbx,QWORD PTR[rdi] + lea rsi,QWORD PTR[48+rdi] + call __mulx_384 + + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[40+rsp] + mov rcx,QWORD PTR[8+rsp] + call __sub_mod_384x384 + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[40+rsp] + lea rdx,QWORD PTR[((40+96))+rsp] + lea rdi,QWORD PTR[40+rsp] + call __sub_mod_384x384 + + lea rbx,QWORD PTR[rcx] + + + lea rsi,QWORD PTR[40+rsp] + mov rcx,QWORD PTR[rsp] + mov rdi,QWORD PTR[32+rsp] + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + + lea rsi,QWORD PTR[((40+192))+rsp] + mov rcx,QWORD PTR[rsp] + lea rdi,QWORD PTR[48+rdi] + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + lea r8,QWORD PTR[328+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_mulx_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mulx_mont_384x:: +mulx_mont_384x ENDP +PUBLIC sqrx_mont_384x + + +ALIGN 32 +sqrx_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_sqrx_mont_384x:: + + + mov QWORD PTR[rsp],rcx + mov rcx,rdx + + mov QWORD PTR[16+rsp],rdi + mov QWORD PTR[24+rsp],rsi + + + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[32+rsp] + call __add_mod_384 + + + mov rsi,QWORD PTR[24+rsp] + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[((32+48))+rsp] + call __sub_mod_384 + + + mov rsi,QWORD PTR[24+rsp] + lea rbx,QWORD PTR[48+rsi] + + mov rdx,QWORD PTR[48+rsi] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov r12,QWORD PTR[24+rsi] + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + lea rsi,QWORD PTR[((-128))+rsi] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r9,r8,r14 + call __mulx_mont_384 + add rdx,rdx + adc r15,r15 + adc rax,rax + mov r8,rdx + adc r12,r12 + mov r9,r15 + adc rdi,rdi + mov r10,rax + adc rbp,rbp + mov r11,r12 + sbb rsi,rsi + + sub rdx,QWORD PTR[rcx] + sbb r15,QWORD PTR[8+rcx] + mov r13,rdi + sbb rax,QWORD PTR[16+rcx] + sbb r12,QWORD PTR[24+rcx] + sbb rdi,QWORD PTR[32+rcx] + mov r14,rbp + sbb rbp,QWORD PTR[40+rcx] + sbb rsi,0 + + cmovc rdx,r8 + cmovc r15,r9 + cmovc rax,r10 + mov QWORD PTR[48+rbx],rdx + cmovc r12,r11 + mov QWORD PTR[56+rbx],r15 + cmovc rdi,r13 + mov QWORD PTR[64+rbx],rax + cmovc rbp,r14 + mov QWORD PTR[72+rbx],r12 + mov QWORD PTR[80+rbx],rdi + mov QWORD PTR[88+rbx],rbp + + lea rsi,QWORD PTR[32+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + + mov rdx,QWORD PTR[((32+48))+rsp] + mov r14,QWORD PTR[((32+0))+rsp] + mov r15,QWORD PTR[((32+8))+rsp] + mov rax,QWORD PTR[((32+16))+rsp] + mov r12,QWORD PTR[((32+24))+rsp] + mov rdi,QWORD PTR[((32+32))+rsp] + mov rbp,QWORD PTR[((32+40))+rsp] + lea rsi,QWORD PTR[((-128))+rsi] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r9,r8,r14 + call __mulx_mont_384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqrx_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_mont_384x:: +sqrx_mont_384x ENDP + +PUBLIC mulx_382x + + +ALIGN 32 +mulx_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mulx_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_mulx_382x:: + + + lea rdi,QWORD PTR[96+rdi] + mov QWORD PTR[rsp],rsi + mov QWORD PTR[8+rsp],rdx + mov QWORD PTR[16+rsp],rdi + mov QWORD PTR[24+rsp],rcx + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + add r8,QWORD PTR[48+rsi] + adc r9,QWORD PTR[56+rsi] + adc r10,QWORD PTR[64+rsi] + adc r11,QWORD PTR[72+rsi] + adc r12,QWORD PTR[80+rsi] + adc r13,QWORD PTR[88+rsi] + + mov QWORD PTR[((32+0))+rsp],r8 + mov QWORD PTR[((32+8))+rsp],r9 + mov QWORD PTR[((32+16))+rsp],r10 + mov QWORD PTR[((32+24))+rsp],r11 + mov QWORD PTR[((32+32))+rsp],r12 + mov QWORD PTR[((32+40))+rsp],r13 + + + mov r8,QWORD PTR[rdx] + mov r9,QWORD PTR[8+rdx] + mov r10,QWORD PTR[16+rdx] + mov r11,QWORD PTR[24+rdx] + mov r12,QWORD PTR[32+rdx] + mov r13,QWORD PTR[40+rdx] + + add r8,QWORD PTR[48+rdx] + adc r9,QWORD PTR[56+rdx] + adc r10,QWORD PTR[64+rdx] + adc r11,QWORD PTR[72+rdx] + adc r12,QWORD PTR[80+rdx] + adc r13,QWORD PTR[88+rdx] + + mov QWORD PTR[((32+48))+rsp],r8 + mov QWORD PTR[((32+56))+rsp],r9 + mov QWORD PTR[((32+64))+rsp],r10 + mov QWORD PTR[((32+72))+rsp],r11 + mov QWORD PTR[((32+80))+rsp],r12 + mov QWORD PTR[((32+88))+rsp],r13 + + + lea rsi,QWORD PTR[((32+0))+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + call __mulx_384 + + + mov rsi,QWORD PTR[rsp] + mov rbx,QWORD PTR[8+rsp] + lea rdi,QWORD PTR[((-96))+rdi] + call __mulx_384 + + + lea rsi,QWORD PTR[((48+128))+rsi] + lea rbx,QWORD PTR[48+rbx] + lea rdi,QWORD PTR[32+rsp] + call __mulx_384 + + + mov rsi,QWORD PTR[16+rsp] + lea rdx,QWORD PTR[32+rsp] + mov rcx,QWORD PTR[24+rsp] + mov rdi,rsi + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[rdi] + lea rdx,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + + lea rsi,QWORD PTR[((-96))+rdi] + lea rdx,QWORD PTR[32+rsp] + lea rdi,QWORD PTR[((-96))+rdi] + call __sub_mod_384x384 + + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_mulx_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mulx_382x:: +mulx_382x ENDP +PUBLIC sqrx_382x + + +ALIGN 32 +sqrx_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rsi + +$L$SEH_body_sqrx_382x:: + + + mov rcx,rdx + + + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov rbx,QWORD PTR[24+rsi] + mov rbp,QWORD PTR[32+rsi] + mov rdx,QWORD PTR[40+rsi] + + mov r8,r14 + add r14,QWORD PTR[48+rsi] + mov r9,r15 + adc r15,QWORD PTR[56+rsi] + mov r10,rax + adc rax,QWORD PTR[64+rsi] + mov r11,rbx + adc rbx,QWORD PTR[72+rsi] + mov r12,rbp + adc rbp,QWORD PTR[80+rsi] + mov r13,rdx + adc rdx,QWORD PTR[88+rsi] + + mov QWORD PTR[rdi],r14 + mov QWORD PTR[8+rdi],r15 + mov QWORD PTR[16+rdi],rax + mov QWORD PTR[24+rdi],rbx + mov QWORD PTR[32+rdi],rbp + mov QWORD PTR[40+rdi],rdx + + + lea rdx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[48+rdi] + call __sub_mod_384_a_is_loaded + + + lea rsi,QWORD PTR[rdi] + lea rbx,QWORD PTR[((-48))+rdi] + lea rdi,QWORD PTR[((-48))+rdi] + call __mulx_384 + + + mov rsi,QWORD PTR[rsp] + lea rbx,QWORD PTR[48+rsi] + lea rdi,QWORD PTR[96+rdi] + call __mulx_384 + + mov r8,QWORD PTR[rdi] + mov r9,QWORD PTR[8+rdi] + mov r10,QWORD PTR[16+rdi] + mov r11,QWORD PTR[24+rdi] + mov r12,QWORD PTR[32+rdi] + mov r13,QWORD PTR[40+rdi] + mov r14,QWORD PTR[48+rdi] + mov r15,QWORD PTR[56+rdi] + mov rax,QWORD PTR[64+rdi] + mov rbx,QWORD PTR[72+rdi] + mov rbp,QWORD PTR[80+rdi] + add r8,r8 + mov rdx,QWORD PTR[88+rdi] + adc r9,r9 + mov QWORD PTR[rdi],r8 + adc r10,r10 + mov QWORD PTR[8+rdi],r9 + adc r11,r11 + mov QWORD PTR[16+rdi],r10 + adc r12,r12 + mov QWORD PTR[24+rdi],r11 + adc r13,r13 + mov QWORD PTR[32+rdi],r12 + adc r14,r14 + mov QWORD PTR[40+rdi],r13 + adc r15,r15 + mov QWORD PTR[48+rdi],r14 + adc rax,rax + mov QWORD PTR[56+rdi],r15 + adc rbx,rbx + mov QWORD PTR[64+rdi],rax + adc rbp,rbp + mov QWORD PTR[72+rdi],rbx + adc rdx,rdx + mov QWORD PTR[80+rdi],rbp + mov QWORD PTR[88+rdi],rdx + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqrx_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_382x:: +sqrx_382x ENDP +PUBLIC mulx_384 + + +ALIGN 32 +mulx_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mulx_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + +$L$SEH_body_mulx_384:: + + + mov rbx,rdx + call __mulx_384 + + mov r15,QWORD PTR[rsp] + + mov r14,QWORD PTR[8+rsp] + + mov r13,QWORD PTR[16+rsp] + + mov r12,QWORD PTR[24+rsp] + + mov rbx,QWORD PTR[32+rsp] + + mov rbp,QWORD PTR[40+rsp] + + lea rsp,QWORD PTR[48+rsp] + +$L$SEH_epilogue_mulx_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mulx_384:: +mulx_384 ENDP + + +ALIGN 32 +__mulx_384 PROC PRIVATE + DB 243,15,30,250 + mov rdx,QWORD PTR[rbx] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + lea rsi,QWORD PTR[((-128))+rsi] + + mulx rcx,r9,r14 + xor rbp,rbp + + mulx rax,r8,r15 + adcx r8,rcx + mov QWORD PTR[rdi],r9 + + mulx rcx,r9,r10 + adcx r9,rax + + mulx rax,r10,r11 + adcx r10,rcx + + mulx rcx,r11,r12 + adcx r11,rax + + mulx r13,r12,r13 + mov rdx,QWORD PTR[8+rbx] + adcx r12,rcx + adcx r13,rbp + mulx rcx,rax,r14 + adcx rax,r8 + adox r9,rcx + mov QWORD PTR[8+rdi],rax + + mulx rcx,r8,r15 + adcx r8,r9 + adox r10,rcx + + mulx rax,r9,QWORD PTR[((128+16))+rsi] + adcx r9,r10 + adox r11,rax + + mulx rcx,r10,QWORD PTR[((128+24))+rsi] + adcx r10,r11 + adox r12,rcx + + mulx rax,r11,QWORD PTR[((128+32))+rsi] + adcx r11,r12 + adox rax,r13 + + mulx r13,r12,QWORD PTR[((128+40))+rsi] + mov rdx,QWORD PTR[16+rbx] + adcx r12,rax + adox r13,rbp + adcx r13,rbp + mulx rcx,rax,r14 + adcx rax,r8 + adox r9,rcx + mov QWORD PTR[16+rdi],rax + + mulx rcx,r8,r15 + adcx r8,r9 + adox r10,rcx + + mulx rax,r9,QWORD PTR[((128+16))+rsi] + adcx r9,r10 + adox r11,rax + + mulx rcx,r10,QWORD PTR[((128+24))+rsi] + adcx r10,r11 + adox r12,rcx + + mulx rax,r11,QWORD PTR[((128+32))+rsi] + adcx r11,r12 + adox rax,r13 + + mulx r13,r12,QWORD PTR[((128+40))+rsi] + mov rdx,QWORD PTR[24+rbx] + adcx r12,rax + adox r13,rbp + adcx r13,rbp + mulx rcx,rax,r14 + adcx rax,r8 + adox r9,rcx + mov QWORD PTR[24+rdi],rax + + mulx rcx,r8,r15 + adcx r8,r9 + adox r10,rcx + + mulx rax,r9,QWORD PTR[((128+16))+rsi] + adcx r9,r10 + adox r11,rax + + mulx rcx,r10,QWORD PTR[((128+24))+rsi] + adcx r10,r11 + adox r12,rcx + + mulx rax,r11,QWORD PTR[((128+32))+rsi] + adcx r11,r12 + adox rax,r13 + + mulx r13,r12,QWORD PTR[((128+40))+rsi] + mov rdx,QWORD PTR[32+rbx] + adcx r12,rax + adox r13,rbp + adcx r13,rbp + mulx rcx,rax,r14 + adcx rax,r8 + adox r9,rcx + mov QWORD PTR[32+rdi],rax + + mulx rcx,r8,r15 + adcx r8,r9 + adox r10,rcx + + mulx rax,r9,QWORD PTR[((128+16))+rsi] + adcx r9,r10 + adox r11,rax + + mulx rcx,r10,QWORD PTR[((128+24))+rsi] + adcx r10,r11 + adox r12,rcx + + mulx rax,r11,QWORD PTR[((128+32))+rsi] + adcx r11,r12 + adox rax,r13 + + mulx r13,r12,QWORD PTR[((128+40))+rsi] + mov rdx,QWORD PTR[40+rbx] + adcx r12,rax + adox r13,rbp + adcx r13,rbp + mulx rcx,rax,r14 + adcx rax,r8 + adox r9,rcx + mov QWORD PTR[40+rdi],rax + + mulx rcx,r8,r15 + adcx r8,r9 + adox r10,rcx + + mulx rax,r9,QWORD PTR[((128+16))+rsi] + adcx r9,r10 + adox r11,rax + + mulx rcx,r10,QWORD PTR[((128+24))+rsi] + adcx r10,r11 + adox r12,rcx + + mulx rax,r11,QWORD PTR[((128+32))+rsi] + adcx r11,r12 + adox rax,r13 + + mulx r13,r12,QWORD PTR[((128+40))+rsi] + mov rdx,rax + adcx r12,rax + adox r13,rbp + adcx r13,rbp + mov QWORD PTR[48+rdi],r8 + mov QWORD PTR[56+rdi],r9 + mov QWORD PTR[64+rdi],r10 + mov QWORD PTR[72+rdi],r11 + mov QWORD PTR[80+rdi],r12 + mov QWORD PTR[88+rdi],r13 + + DB 0F3h,0C3h ;repret +__mulx_384 ENDP +PUBLIC sqrx_384 + + +ALIGN 32 +sqrx_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_384:: + mov rdi,rcx + mov rsi,rdx + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + push rdi + +$L$SEH_body_sqrx_384:: + + + call __sqrx_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sqrx_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_384:: +sqrx_384 ENDP + +ALIGN 32 +__sqrx_384 PROC PRIVATE + DB 243,15,30,250 + mov rdx,QWORD PTR[rsi] + mov r14,QWORD PTR[8+rsi] + mov r15,QWORD PTR[16+rsi] + mov rcx,QWORD PTR[24+rsi] + mov rbx,QWORD PTR[32+rsi] + + + mulx rdi,r8,r14 + mov rbp,QWORD PTR[40+rsi] + mulx rax,r9,r15 + add r9,rdi + mulx rdi,r10,rcx + adc r10,rax + mulx rax,r11,rbx + adc r11,rdi + mulx r13,r12,rbp + mov rdx,r14 + adc r12,rax + adc r13,0 + + + xor r14,r14 + mulx rax,rdi,r15 + adcx r10,rdi + adox r11,rax + + mulx rax,rdi,rcx + adcx r11,rdi + adox r12,rax + + mulx rax,rdi,rbx + adcx r12,rdi + adox r13,rax + + mulx rax,rdi,rbp + mov rdx,r15 + adcx r13,rdi + adox rax,r14 + adcx r14,rax + + + xor r15,r15 + mulx rax,rdi,rcx + adcx r12,rdi + adox r13,rax + + mulx rax,rdi,rbx + adcx r13,rdi + adox r14,rax + + mulx rax,rdi,rbp + mov rdx,rcx + adcx r14,rdi + adox rax,r15 + adcx r15,rax + + + xor rcx,rcx + mulx rax,rdi,rbx + adcx r14,rdi + adox r15,rax + + mulx rax,rdi,rbp + mov rdx,rbx + adcx r15,rdi + adox rax,rcx + adcx rcx,rax + + + mulx rbx,rdi,rbp + mov rdx,QWORD PTR[rsi] + add rcx,rdi + mov rdi,QWORD PTR[8+rsp] + adc rbx,0 + + + xor rbp,rbp + adcx r8,r8 + adcx r9,r9 + adcx r10,r10 + adcx r11,r11 + adcx r12,r12 + + + mulx rax,rdx,rdx + mov QWORD PTR[rdi],rdx + mov rdx,QWORD PTR[8+rsi] + adox r8,rax + mov QWORD PTR[8+rdi],r8 + + mulx rax,r8,rdx + mov rdx,QWORD PTR[16+rsi] + adox r9,r8 + adox r10,rax + mov QWORD PTR[16+rdi],r9 + mov QWORD PTR[24+rdi],r10 + + mulx r9,r8,rdx + mov rdx,QWORD PTR[24+rsi] + adox r11,r8 + adox r12,r9 + adcx r13,r13 + adcx r14,r14 + mov QWORD PTR[32+rdi],r11 + mov QWORD PTR[40+rdi],r12 + + mulx r9,r8,rdx + mov rdx,QWORD PTR[32+rsi] + adox r13,r8 + adox r14,r9 + adcx r15,r15 + adcx rcx,rcx + mov QWORD PTR[48+rdi],r13 + mov QWORD PTR[56+rdi],r14 + + mulx r9,r8,rdx + mov rdx,QWORD PTR[40+rsi] + adox r15,r8 + adox rcx,r9 + adcx rbx,rbx + adcx rbp,rbp + mov QWORD PTR[64+rdi],r15 + mov QWORD PTR[72+rdi],rcx + + mulx r9,r8,rdx + adox rbx,r8 + adox rbp,r9 + + mov QWORD PTR[80+rdi],rbx + mov QWORD PTR[88+rdi],rbp + + DB 0F3h,0C3h ;repret +__sqrx_384 ENDP + + + +PUBLIC redcx_mont_384 + + +ALIGN 32 +redcx_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_redcx_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_redcx_mont_384:: + + + mov rbx,rdx + call __mulx_by_1_mont_384 + call __redc_tail_mont_384 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_redcx_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_redcx_mont_384:: +redcx_mont_384 ENDP + + + + +PUBLIC fromx_mont_384 + + +ALIGN 32 +fromx_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_fromx_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_fromx_mont_384:: + + + mov rbx,rdx + call __mulx_by_1_mont_384 + + + + + mov rax,r14 + mov rcx,r15 + mov rdx,r8 + mov rbp,r9 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + mov r13,r10 + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + mov rsi,r11 + sbb r11,QWORD PTR[40+rbx] + + cmovc r14,rax + cmovc r15,rcx + cmovc r8,rdx + mov QWORD PTR[rdi],r14 + cmovc r9,rbp + mov QWORD PTR[8+rdi],r15 + cmovc r10,r13 + mov QWORD PTR[16+rdi],r8 + cmovc r11,rsi + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_fromx_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_fromx_mont_384:: +fromx_mont_384 ENDP + +ALIGN 32 +__mulx_by_1_mont_384 PROC PRIVATE + DB 243,15,30,250 + mov r8,QWORD PTR[rsi] + mov rdx,rcx + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + imul rdx,r8 + + + xor r14,r14 + mulx rbp,rax,QWORD PTR[rbx] + adcx r8,rax + adox r9,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r9,rax + adox r10,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r10,rax + adox r11,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r11,rax + adox r12,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r12,rax + adox r13,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r13,rax + adox rbp,r14 + adcx r14,rbp + imul rdx,r9 + + + xor r15,r15 + mulx rbp,rax,QWORD PTR[rbx] + adcx r9,rax + adox r10,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r10,rax + adox r11,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r11,rax + adox r12,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r12,rax + adox r13,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r13,rax + adox r14,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r14,rax + adox rbp,r15 + adcx r15,rbp + imul rdx,r10 + + + xor r8,r8 + mulx rbp,rax,QWORD PTR[rbx] + adcx r10,rax + adox r11,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r11,rax + adox r12,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r12,rax + adox r13,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r13,rax + adox r14,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r14,rax + adox r15,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r15,rax + adox rbp,r8 + adcx r8,rbp + imul rdx,r11 + + + xor r9,r9 + mulx rbp,rax,QWORD PTR[rbx] + adcx r11,rax + adox r12,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r12,rax + adox r13,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r13,rax + adox r14,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r14,rax + adox r15,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r15,rax + adox r8,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r8,rax + adox rbp,r9 + adcx r9,rbp + imul rdx,r12 + + + xor r10,r10 + mulx rbp,rax,QWORD PTR[rbx] + adcx r12,rax + adox r13,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r13,rax + adox r14,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r14,rax + adox r15,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r15,rax + adox r8,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r8,rax + adox r9,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r9,rax + adox rbp,r10 + adcx r10,rbp + imul rdx,r13 + + + xor r11,r11 + mulx rbp,rax,QWORD PTR[rbx] + adcx r13,rax + adox r14,rbp + + mulx rbp,rax,QWORD PTR[8+rbx] + adcx r14,rax + adox r15,rbp + + mulx rbp,rax,QWORD PTR[16+rbx] + adcx r15,rax + adox r8,rbp + + mulx rbp,rax,QWORD PTR[24+rbx] + adcx r8,rax + adox r9,rbp + + mulx rbp,rax,QWORD PTR[32+rbx] + adcx r9,rax + adox r10,rbp + + mulx rbp,rax,QWORD PTR[40+rbx] + mov rdx,rcx + adcx r10,rax + adox rbp,r11 + adcx r11,rbp + DB 0F3h,0C3h ;repret +__mulx_by_1_mont_384 ENDP + + +ALIGN 32 +__redc_tail_mont_384 PROC PRIVATE + DB 243,15,30,250 + add r14,QWORD PTR[48+rsi] + mov rax,r14 + adc r15,QWORD PTR[56+rsi] + adc r8,QWORD PTR[64+rsi] + adc r9,QWORD PTR[72+rsi] + mov rcx,r15 + adc r10,QWORD PTR[80+rsi] + adc r11,QWORD PTR[88+rsi] + sbb r12,r12 + + + + + mov rdx,r8 + mov rbp,r9 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + mov r13,r10 + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + mov rsi,r11 + sbb r11,QWORD PTR[40+rbx] + sbb r12,0 + + cmovc r14,rax + cmovc r15,rcx + cmovc r8,rdx + mov QWORD PTR[rdi],r14 + cmovc r9,rbp + mov QWORD PTR[8+rdi],r15 + cmovc r10,r13 + mov QWORD PTR[16+rdi],r8 + cmovc r11,rsi + mov QWORD PTR[24+rdi],r9 + mov QWORD PTR[32+rdi],r10 + mov QWORD PTR[40+rdi],r11 + + DB 0F3h,0C3h ;repret +__redc_tail_mont_384 ENDP + +PUBLIC sgn0x_pty_mont_384 + + +ALIGN 32 +sgn0x_pty_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0x_pty_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sgn0x_pty_mont_384:: + + + mov rbx,rsi + lea rsi,QWORD PTR[rdi] + mov rcx,rdx + call __mulx_by_1_mont_384 + + xor rax,rax + mov r13,r14 + add r14,r14 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rax,0 + + sub r14,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rax,0 + + not rax + and r13,1 + and rax,2 + or rax,r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sgn0x_pty_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0x_pty_mont_384:: +sgn0x_pty_mont_384 ENDP + +PUBLIC sgn0x_pty_mont_384x + + +ALIGN 32 +sgn0x_pty_mont_384x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sgn0x_pty_mont_384x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,8 + +$L$SEH_body_sgn0x_pty_mont_384x:: + + + mov rbx,rsi + lea rsi,QWORD PTR[48+rdi] + mov rcx,rdx + call __mulx_by_1_mont_384 + + mov r12,r14 + or r14,r15 + or r14,r8 + or r14,r9 + or r14,r10 + or r14,r11 + + lea rsi,QWORD PTR[rdi] + xor rdi,rdi + mov r13,r12 + add r12,r12 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rdi,0 + + sub r12,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rdi,0 + + mov QWORD PTR[rsp],r14 + not rdi + and r13,1 + and rdi,2 + or rdi,r13 + + call __mulx_by_1_mont_384 + + mov r12,r14 + or r14,r15 + or r14,r8 + or r14,r9 + or r14,r10 + or r14,r11 + + xor rax,rax + mov r13,r12 + add r12,r12 + adc r15,r15 + adc r8,r8 + adc r9,r9 + adc r10,r10 + adc r11,r11 + adc rax,0 + + sub r12,QWORD PTR[rbx] + sbb r15,QWORD PTR[8+rbx] + sbb r8,QWORD PTR[16+rbx] + sbb r9,QWORD PTR[24+rbx] + sbb r10,QWORD PTR[32+rbx] + sbb r11,QWORD PTR[40+rbx] + sbb rax,0 + + mov r12,QWORD PTR[rsp] + + not rax + + test r14,r14 + cmovz r13,rdi + + test r12,r12 + cmovnz rax,rdi + + and r13,1 + and rax,2 + or rax,r13 + + mov r15,QWORD PTR[8+rsp] + + mov r14,QWORD PTR[16+rsp] + + mov r13,QWORD PTR[24+rsp] + + mov r12,QWORD PTR[32+rsp] + + mov rbx,QWORD PTR[40+rsp] + + mov rbp,QWORD PTR[48+rsp] + + lea rsp,QWORD PTR[56+rsp] + +$L$SEH_epilogue_sgn0x_pty_mont_384x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sgn0x_pty_mont_384x:: +sgn0x_pty_mont_384x ENDP +PUBLIC mulx_mont_384 + + +ALIGN 32 +mulx_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_mulx_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + lea rsp,QWORD PTR[((-24))+rsp] + +$L$SEH_body_mulx_mont_384:: + + + mov rbx,rdx + mov rdx,QWORD PTR[rdx] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov r12,QWORD PTR[24+rsi] + mov QWORD PTR[16+rsp],rdi + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + lea rsi,QWORD PTR[((-128))+rsi] + lea rcx,QWORD PTR[((-128))+rcx] + mov QWORD PTR[rsp],r8 + + mulx r9,r8,r14 + call __mulx_mont_384 + + mov r15,QWORD PTR[24+rsp] + + mov r14,QWORD PTR[32+rsp] + + mov r13,QWORD PTR[40+rsp] + + mov r12,QWORD PTR[48+rsp] + + mov rbx,QWORD PTR[56+rsp] + + mov rbp,QWORD PTR[64+rsp] + + lea rsp,QWORD PTR[72+rsp] + +$L$SEH_epilogue_mulx_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_mulx_mont_384:: +mulx_mont_384 ENDP + +ALIGN 32 +__mulx_mont_384 PROC PRIVATE + DB 243,15,30,250 + + mulx r10,r14,r15 + mulx r11,r15,rax + add r9,r14 + mulx r12,rax,r12 + adc r10,r15 + mulx r13,rdi,rdi + adc r11,rax + mulx r14,rbp,rbp + mov rdx,QWORD PTR[8+rbx] + adc r12,rdi + adc r13,rbp + adc r14,0 + xor r15,r15 + + mov QWORD PTR[16+rsp],r8 + imul r8,QWORD PTR[8+rsp] + + + xor rax,rax + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r9,rdi + adcx r10,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r10,rdi + adcx r11,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r8 + adox r14,rdi + adcx r15,rbp + adox r15,rax + adox rax,rax + + + xor r8,r8 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rdi,QWORD PTR[16+rsp] + adox r9,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r9,rdi + adox r10,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r10,rdi + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[16+rbx] + adcx r13,rdi + adox r14,rbp + adcx r14,r8 + adox r15,r8 + adcx r15,r8 + adox rax,r8 + adcx rax,r8 + mov QWORD PTR[16+rsp],r9 + imul r9,QWORD PTR[8+rsp] + + + xor r8,r8 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r10,rdi + adcx r11,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r9 + adox r15,rdi + adcx rax,rbp + adox rax,r8 + adox r8,r8 + + + xor r9,r9 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rdi,QWORD PTR[16+rsp] + adox r10,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r10,rdi + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[24+rbx] + adcx r14,rdi + adox r15,rbp + adcx r15,r9 + adox rax,r9 + adcx rax,r9 + adox r8,r9 + adcx r8,r9 + mov QWORD PTR[16+rsp],r10 + imul r10,QWORD PTR[8+rsp] + + + xor r9,r9 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r10 + adox rax,rdi + adcx r8,rbp + adox r8,r9 + adox r9,r9 + + + xor r10,r10 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rdi,QWORD PTR[16+rsp] + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[32+rbx] + adcx r15,rdi + adox rax,rbp + adcx rax,r10 + adox r8,r10 + adcx r8,r10 + adox r9,r10 + adcx r9,r10 + mov QWORD PTR[16+rsp],r11 + imul r11,QWORD PTR[8+rsp] + + + xor r10,r10 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox rax,rdi + adcx r8,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r11 + adox r8,rdi + adcx r9,rbp + adox r9,r10 + adox r10,r10 + + + xor r11,r11 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rdi,QWORD PTR[16+rsp] + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[40+rbx] + adcx rax,rdi + adox r8,rbp + adcx r8,r11 + adox r9,r11 + adcx r9,r11 + adox r10,r11 + adcx r10,r11 + mov QWORD PTR[16+rsp],r12 + imul r12,QWORD PTR[8+rsp] + + + xor r11,r11 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox rax,rdi + adcx r8,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r8,rdi + adcx r9,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r12 + adox r9,rdi + adcx r10,rbp + adox r10,r11 + adox r11,r11 + + + xor r12,r12 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rdi,QWORD PTR[16+rsp] + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx rax,rdi + adox r8,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,r13 + adcx r8,rdi + adox r9,rbp + adcx r9,r12 + adox r10,r12 + adcx r10,r12 + adox r11,r12 + adcx r11,r12 + imul rdx,QWORD PTR[8+rsp] + mov rbx,QWORD PTR[24+rsp] + + + xor r12,r12 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx rax,rdi + adox r8,rbp + mov r13,r15 + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r8,rdi + adox r9,rbp + mov rsi,rax + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + adcx r9,rdi + adox r10,rbp + mov rdx,r14 + adcx r10,r12 + adox r11,r12 + lea rcx,QWORD PTR[128+rcx] + mov r12,r8 + adc r11,0 + + + + + sub r14,QWORD PTR[rcx] + sbb r15,QWORD PTR[8+rcx] + mov rdi,r9 + sbb rax,QWORD PTR[16+rcx] + sbb r8,QWORD PTR[24+rcx] + sbb r9,QWORD PTR[32+rcx] + mov rbp,r10 + sbb r10,QWORD PTR[40+rcx] + sbb r11,0 + + cmovnc rdx,r14 + cmovc r15,r13 + cmovc rax,rsi + cmovnc r12,r8 + mov QWORD PTR[rbx],rdx + cmovnc rdi,r9 + mov QWORD PTR[8+rbx],r15 + cmovnc rbp,r10 + mov QWORD PTR[16+rbx],rax + mov QWORD PTR[24+rbx],r12 + mov QWORD PTR[32+rbx],rdi + mov QWORD PTR[40+rbx],rbp + + DB 0F3h,0C3h ;repret + +__mulx_mont_384 ENDP +PUBLIC sqrx_mont_384 + + +ALIGN 32 +sqrx_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + lea rsp,QWORD PTR[((-24))+rsp] + +$L$SEH_body_sqrx_mont_384:: + + + mov r8,rcx + lea rcx,QWORD PTR[((-128))+rdx] + mov rdx,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov r12,QWORD PTR[24+rsi] + mov QWORD PTR[16+rsp],rdi + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + + lea rbx,QWORD PTR[rsi] + mov QWORD PTR[rsp],r8 + lea rsi,QWORD PTR[((-128))+rsi] + + mulx r9,r8,rdx + call __mulx_mont_384 + + mov r15,QWORD PTR[24+rsp] + + mov r14,QWORD PTR[32+rsp] + + mov r13,QWORD PTR[40+rsp] + + mov r12,QWORD PTR[48+rsp] + + mov rbx,QWORD PTR[56+rsp] + + mov rbp,QWORD PTR[64+rsp] + + lea rsp,QWORD PTR[72+rsp] + +$L$SEH_epilogue_sqrx_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_mont_384:: +sqrx_mont_384 ENDP + +PUBLIC sqrx_n_mul_mont_384 + + +ALIGN 32 +sqrx_n_mul_mont_384 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_n_mul_mont_384:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + mov r9,QWORD PTR[48+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + lea rsp,QWORD PTR[((-40))+rsp] + +$L$SEH_body_sqrx_n_mul_mont_384:: + + + mov r10,rdx + mov rdx,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov rbx,rsi + mov r12,QWORD PTR[24+rsi] + mov QWORD PTR[16+rsp],rdi + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + + mov QWORD PTR[rsp],r8 + mov QWORD PTR[24+rsp],r9 + movq xmm2,QWORD PTR[r9] + +$L$oop_sqrx_384:: + movd xmm1,r10d + lea rsi,QWORD PTR[((-128))+rbx] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r9,r8,rdx + call __mulx_mont_384 + + movd r10d,xmm1 + dec r10d + jnz $L$oop_sqrx_384 + + mov r14,rdx +DB 102,72,15,126,210 + lea rsi,QWORD PTR[((-128))+rbx] + mov rbx,QWORD PTR[24+rsp] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r9,r8,r14 + call __mulx_mont_384 + + mov r15,QWORD PTR[40+rsp] + + mov r14,QWORD PTR[48+rsp] + + mov r13,QWORD PTR[56+rsp] + + mov r12,QWORD PTR[64+rsp] + + mov rbx,QWORD PTR[72+rsp] + + mov rbp,QWORD PTR[80+rsp] + + lea rsp,QWORD PTR[88+rsp] + +$L$SEH_epilogue_sqrx_n_mul_mont_384:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_n_mul_mont_384:: +sqrx_n_mul_mont_384 ENDP + +PUBLIC sqrx_n_mul_mont_383 + + +ALIGN 32 +sqrx_n_mul_mont_383 PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_n_mul_mont_383:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + mov r8,QWORD PTR[40+rsp] + mov r9,QWORD PTR[48+rsp] + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + lea rsp,QWORD PTR[((-40))+rsp] + +$L$SEH_body_sqrx_n_mul_mont_383:: + + + mov r10,rdx + mov rdx,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov rbx,rsi + mov r12,QWORD PTR[24+rsi] + mov QWORD PTR[16+rsp],rdi + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + + mov QWORD PTR[rsp],r8 + mov QWORD PTR[24+rsp],r9 + movq xmm2,QWORD PTR[r9] + lea rcx,QWORD PTR[((-128))+rcx] + +$L$oop_sqrx_383:: + movd xmm1,r10d + lea rsi,QWORD PTR[((-128))+rbx] + + mulx r9,r8,rdx + call __mulx_mont_383_nonred + + movd r10d,xmm1 + dec r10d + jnz $L$oop_sqrx_383 + + mov r14,rdx +DB 102,72,15,126,210 + lea rsi,QWORD PTR[((-128))+rbx] + mov rbx,QWORD PTR[24+rsp] + + mulx r9,r8,r14 + call __mulx_mont_384 + + mov r15,QWORD PTR[40+rsp] + + mov r14,QWORD PTR[48+rsp] + + mov r13,QWORD PTR[56+rsp] + + mov r12,QWORD PTR[64+rsp] + + mov rbx,QWORD PTR[72+rsp] + + mov rbp,QWORD PTR[80+rsp] + + lea rsp,QWORD PTR[88+rsp] + +$L$SEH_epilogue_sqrx_n_mul_mont_383:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_n_mul_mont_383:: +sqrx_n_mul_mont_383 ENDP + +ALIGN 32 +__mulx_mont_383_nonred PROC PRIVATE + DB 243,15,30,250 + + mulx r10,r14,r15 + mulx r11,r15,rax + add r9,r14 + mulx r12,rax,r12 + adc r10,r15 + mulx r13,rdi,rdi + adc r11,rax + mulx r14,rbp,rbp + mov rdx,QWORD PTR[8+rbx] + adc r12,rdi + adc r13,rbp + adc r14,0 + mov rax,r8 + imul r8,QWORD PTR[8+rsp] + + + xor r15,r15 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r9,rdi + adcx r10,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r10,rdi + adcx r11,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r8 + adox r14,rdi + adcx rbp,r15 + adox r15,rbp + + + xor r8,r8 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx rax,rdi + adox r9,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r9,rdi + adox r10,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r10,rdi + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[16+rbx] + adcx r13,rdi + adox r14,rbp + adcx r14,rax + adox r15,rax + adcx r15,rax + mov r8,r9 + imul r9,QWORD PTR[8+rsp] + + + xor rax,rax + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r10,rdi + adcx r11,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r9 + adox r15,rdi + adcx rbp,rax + adox rax,rbp + + + xor r9,r9 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r8,rdi + adox r10,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r10,rdi + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[24+rbx] + adcx r14,rdi + adox r15,rbp + adcx r15,r8 + adox rax,r8 + adcx rax,r8 + mov r9,r10 + imul r10,QWORD PTR[8+rsp] + + + xor r8,r8 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r11,rdi + adcx r12,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r10 + adox rax,rdi + adcx rbp,r8 + adox r8,rbp + + + xor r10,r10 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r9,rdi + adox r11,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r11,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[32+rbx] + adcx r15,rdi + adox rax,rbp + adcx rax,r9 + adox r8,r9 + adcx r8,r9 + mov r10,r11 + imul r11,QWORD PTR[8+rsp] + + + xor r9,r9 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r12,rdi + adcx r13,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox rax,rdi + adcx r8,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r11 + adox r8,rdi + adcx rbp,r9 + adox r9,rbp + + + xor r11,r11 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r10,rdi + adox r12,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r12,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,QWORD PTR[40+rbx] + adcx rax,rdi + adox r8,rbp + adcx r8,r10 + adox r9,r10 + adcx r9,r10 + mov r11,r12 + imul r12,QWORD PTR[8+rsp] + + + xor r10,r10 + mulx rbp,rdi,QWORD PTR[((0+128))+rsi] + adox r13,rdi + adcx r14,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rsi] + adox r14,rdi + adcx r15,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rsi] + adox r15,rdi + adcx rax,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rsi] + adox rax,rdi + adcx r8,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rsi] + adox r8,rdi + adcx r9,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rsi] + mov rdx,r12 + adox r9,rdi + adcx rbp,r10 + adox r10,rbp + + + xor r12,r12 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r11,rdi + adox r13,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx rax,rdi + adox r8,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,r13 + adcx r8,rdi + adox r9,rbp + adcx r9,r11 + adox r10,r11 + adcx r10,r11 + imul rdx,QWORD PTR[8+rsp] + mov rbx,QWORD PTR[24+rsp] + + + xor r12,r12 + mulx rbp,rdi,QWORD PTR[((0+128))+rcx] + adcx r13,rdi + adox r14,rbp + + mulx rbp,rdi,QWORD PTR[((8+128))+rcx] + adcx r14,rdi + adox r15,rbp + + mulx rbp,rdi,QWORD PTR[((16+128))+rcx] + adcx r15,rdi + adox rax,rbp + + mulx rbp,rdi,QWORD PTR[((24+128))+rcx] + adcx rax,rdi + adox r8,rbp + + mulx rbp,rdi,QWORD PTR[((32+128))+rcx] + adcx r8,rdi + adox r9,rbp + + mulx rbp,rdi,QWORD PTR[((40+128))+rcx] + mov rdx,r14 + adcx r9,rdi + adox r10,rbp + adc r10,0 + mov r12,r8 + + mov QWORD PTR[rbx],r14 + mov QWORD PTR[8+rbx],r15 + mov QWORD PTR[16+rbx],rax + mov rdi,r9 + mov QWORD PTR[24+rbx],r8 + mov QWORD PTR[32+rbx],r9 + mov QWORD PTR[40+rbx],r10 + mov rbp,r10 + + DB 0F3h,0C3h ;repret + +__mulx_mont_383_nonred ENDP +PUBLIC sqrx_mont_382x + + +ALIGN 32 +sqrx_mont_382x PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_sqrx_mont_382x:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + mov rcx,r9 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,136 + +$L$SEH_body_sqrx_mont_382x:: + + + mov QWORD PTR[rsp],rcx + mov rcx,rdx + mov QWORD PTR[16+rsp],rdi + mov QWORD PTR[24+rsp],rsi + + + mov r8,QWORD PTR[rsi] + mov r9,QWORD PTR[8+rsi] + mov r10,QWORD PTR[16+rsi] + mov r11,QWORD PTR[24+rsi] + mov r12,QWORD PTR[32+rsi] + mov r13,QWORD PTR[40+rsi] + + mov r14,r8 + add r8,QWORD PTR[48+rsi] + mov r15,r9 + adc r9,QWORD PTR[56+rsi] + mov rax,r10 + adc r10,QWORD PTR[64+rsi] + mov rdx,r11 + adc r11,QWORD PTR[72+rsi] + mov rbx,r12 + adc r12,QWORD PTR[80+rsi] + mov rbp,r13 + adc r13,QWORD PTR[88+rsi] + + sub r14,QWORD PTR[48+rsi] + sbb r15,QWORD PTR[56+rsi] + sbb rax,QWORD PTR[64+rsi] + sbb rdx,QWORD PTR[72+rsi] + sbb rbx,QWORD PTR[80+rsi] + sbb rbp,QWORD PTR[88+rsi] + sbb rdi,rdi + + mov QWORD PTR[((32+0))+rsp],r8 + mov QWORD PTR[((32+8))+rsp],r9 + mov QWORD PTR[((32+16))+rsp],r10 + mov QWORD PTR[((32+24))+rsp],r11 + mov QWORD PTR[((32+32))+rsp],r12 + mov QWORD PTR[((32+40))+rsp],r13 + + mov QWORD PTR[((32+48))+rsp],r14 + mov QWORD PTR[((32+56))+rsp],r15 + mov QWORD PTR[((32+64))+rsp],rax + mov QWORD PTR[((32+72))+rsp],rdx + mov QWORD PTR[((32+80))+rsp],rbx + mov QWORD PTR[((32+88))+rsp],rbp + mov QWORD PTR[((32+96))+rsp],rdi + + + + lea rbx,QWORD PTR[48+rsi] + + mov rdx,QWORD PTR[48+rsi] + mov r14,QWORD PTR[rsi] + mov r15,QWORD PTR[8+rsi] + mov rax,QWORD PTR[16+rsi] + mov r12,QWORD PTR[24+rsi] + mov rdi,QWORD PTR[32+rsi] + mov rbp,QWORD PTR[40+rsi] + lea rsi,QWORD PTR[((-128))+rsi] + lea rcx,QWORD PTR[((-128))+rcx] + + mulx r9,r8,r14 + call __mulx_mont_383_nonred + add rdx,rdx + adc r15,r15 + adc rax,rax + adc r12,r12 + adc rdi,rdi + adc rbp,rbp + + mov QWORD PTR[48+rbx],rdx + mov QWORD PTR[56+rbx],r15 + mov QWORD PTR[64+rbx],rax + mov QWORD PTR[72+rbx],r12 + mov QWORD PTR[80+rbx],rdi + mov QWORD PTR[88+rbx],rbp + + lea rsi,QWORD PTR[((32-128))+rsp] + lea rbx,QWORD PTR[((32+48))+rsp] + + mov rdx,QWORD PTR[((32+48))+rsp] + mov r14,QWORD PTR[((32+0))+rsp] + mov r15,QWORD PTR[((32+8))+rsp] + mov rax,QWORD PTR[((32+16))+rsp] + mov r12,QWORD PTR[((32+24))+rsp] + mov rdi,QWORD PTR[((32+32))+rsp] + mov rbp,QWORD PTR[((32+40))+rsp] + + + + mulx r9,r8,r14 + call __mulx_mont_383_nonred + mov r14,QWORD PTR[((32+96))+rsp] + lea rcx,QWORD PTR[128+rcx] + mov r8,QWORD PTR[((32+0))+rsp] + and r8,r14 + mov r9,QWORD PTR[((32+8))+rsp] + and r9,r14 + mov r10,QWORD PTR[((32+16))+rsp] + and r10,r14 + mov r11,QWORD PTR[((32+24))+rsp] + and r11,r14 + mov r13,QWORD PTR[((32+32))+rsp] + and r13,r14 + and r14,QWORD PTR[((32+40))+rsp] + + sub rdx,r8 + mov r8,QWORD PTR[rcx] + sbb r15,r9 + mov r9,QWORD PTR[8+rcx] + sbb rax,r10 + mov r10,QWORD PTR[16+rcx] + sbb r12,r11 + mov r11,QWORD PTR[24+rcx] + sbb rdi,r13 + mov r13,QWORD PTR[32+rcx] + sbb rbp,r14 + sbb r14,r14 + + and r8,r14 + and r9,r14 + and r10,r14 + and r11,r14 + and r13,r14 + and r14,QWORD PTR[40+rcx] + + add rdx,r8 + adc r15,r9 + adc rax,r10 + adc r12,r11 + adc rdi,r13 + adc rbp,r14 + + mov QWORD PTR[rbx],rdx + mov QWORD PTR[8+rbx],r15 + mov QWORD PTR[16+rbx],rax + mov QWORD PTR[24+rbx],r12 + mov QWORD PTR[32+rbx],rdi + mov QWORD PTR[40+rbx],rbp + lea r8,QWORD PTR[136+rsp] + mov r15,QWORD PTR[r8] + + mov r14,QWORD PTR[8+r8] + + mov r13,QWORD PTR[16+r8] + + mov r12,QWORD PTR[24+r8] + + mov rbx,QWORD PTR[32+r8] + + mov rbp,QWORD PTR[40+r8] + + lea rsp,QWORD PTR[48+r8] + +$L$SEH_epilogue_sqrx_mont_382x:: + mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue + mov rsi,QWORD PTR[16+rsp] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_sqrx_mont_382x:: +sqrx_mont_382x ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mulx_mont_384x + DD imagerel $L$SEH_body_mulx_mont_384x + DD imagerel $L$SEH_info_mulx_mont_384x_prologue + + DD imagerel $L$SEH_body_mulx_mont_384x + DD imagerel $L$SEH_epilogue_mulx_mont_384x + DD imagerel $L$SEH_info_mulx_mont_384x_body + + DD imagerel $L$SEH_epilogue_mulx_mont_384x + DD imagerel $L$SEH_end_mulx_mont_384x + DD imagerel $L$SEH_info_mulx_mont_384x_epilogue + + DD imagerel $L$SEH_begin_sqrx_mont_384x + DD imagerel $L$SEH_body_sqrx_mont_384x + DD imagerel $L$SEH_info_sqrx_mont_384x_prologue + + DD imagerel $L$SEH_body_sqrx_mont_384x + DD imagerel $L$SEH_epilogue_sqrx_mont_384x + DD imagerel $L$SEH_info_sqrx_mont_384x_body + + DD imagerel $L$SEH_epilogue_sqrx_mont_384x + DD imagerel $L$SEH_end_sqrx_mont_384x + DD imagerel $L$SEH_info_sqrx_mont_384x_epilogue + + DD imagerel $L$SEH_begin_mulx_382x + DD imagerel $L$SEH_body_mulx_382x + DD imagerel $L$SEH_info_mulx_382x_prologue + + DD imagerel $L$SEH_body_mulx_382x + DD imagerel $L$SEH_epilogue_mulx_382x + DD imagerel $L$SEH_info_mulx_382x_body + + DD imagerel $L$SEH_epilogue_mulx_382x + DD imagerel $L$SEH_end_mulx_382x + DD imagerel $L$SEH_info_mulx_382x_epilogue + + DD imagerel $L$SEH_begin_sqrx_382x + DD imagerel $L$SEH_body_sqrx_382x + DD imagerel $L$SEH_info_sqrx_382x_prologue + + DD imagerel $L$SEH_body_sqrx_382x + DD imagerel $L$SEH_epilogue_sqrx_382x + DD imagerel $L$SEH_info_sqrx_382x_body + + DD imagerel $L$SEH_epilogue_sqrx_382x + DD imagerel $L$SEH_end_sqrx_382x + DD imagerel $L$SEH_info_sqrx_382x_epilogue + + DD imagerel $L$SEH_begin_mulx_384 + DD imagerel $L$SEH_body_mulx_384 + DD imagerel $L$SEH_info_mulx_384_prologue + + DD imagerel $L$SEH_body_mulx_384 + DD imagerel $L$SEH_epilogue_mulx_384 + DD imagerel $L$SEH_info_mulx_384_body + + DD imagerel $L$SEH_epilogue_mulx_384 + DD imagerel $L$SEH_end_mulx_384 + DD imagerel $L$SEH_info_mulx_384_epilogue + + DD imagerel $L$SEH_begin_sqrx_384 + DD imagerel $L$SEH_body_sqrx_384 + DD imagerel $L$SEH_info_sqrx_384_prologue + + DD imagerel $L$SEH_body_sqrx_384 + DD imagerel $L$SEH_epilogue_sqrx_384 + DD imagerel $L$SEH_info_sqrx_384_body + + DD imagerel $L$SEH_epilogue_sqrx_384 + DD imagerel $L$SEH_end_sqrx_384 + DD imagerel $L$SEH_info_sqrx_384_epilogue + + DD imagerel $L$SEH_begin_redcx_mont_384 + DD imagerel $L$SEH_body_redcx_mont_384 + DD imagerel $L$SEH_info_redcx_mont_384_prologue + + DD imagerel $L$SEH_body_redcx_mont_384 + DD imagerel $L$SEH_epilogue_redcx_mont_384 + DD imagerel $L$SEH_info_redcx_mont_384_body + + DD imagerel $L$SEH_epilogue_redcx_mont_384 + DD imagerel $L$SEH_end_redcx_mont_384 + DD imagerel $L$SEH_info_redcx_mont_384_epilogue + + DD imagerel $L$SEH_begin_fromx_mont_384 + DD imagerel $L$SEH_body_fromx_mont_384 + DD imagerel $L$SEH_info_fromx_mont_384_prologue + + DD imagerel $L$SEH_body_fromx_mont_384 + DD imagerel $L$SEH_epilogue_fromx_mont_384 + DD imagerel $L$SEH_info_fromx_mont_384_body + + DD imagerel $L$SEH_epilogue_fromx_mont_384 + DD imagerel $L$SEH_end_fromx_mont_384 + DD imagerel $L$SEH_info_fromx_mont_384_epilogue + + DD imagerel $L$SEH_begin_sgn0x_pty_mont_384 + DD imagerel $L$SEH_body_sgn0x_pty_mont_384 + DD imagerel $L$SEH_info_sgn0x_pty_mont_384_prologue + + DD imagerel $L$SEH_body_sgn0x_pty_mont_384 + DD imagerel $L$SEH_epilogue_sgn0x_pty_mont_384 + DD imagerel $L$SEH_info_sgn0x_pty_mont_384_body + + DD imagerel $L$SEH_epilogue_sgn0x_pty_mont_384 + DD imagerel $L$SEH_end_sgn0x_pty_mont_384 + DD imagerel $L$SEH_info_sgn0x_pty_mont_384_epilogue + + DD imagerel $L$SEH_begin_sgn0x_pty_mont_384x + DD imagerel $L$SEH_body_sgn0x_pty_mont_384x + DD imagerel $L$SEH_info_sgn0x_pty_mont_384x_prologue + + DD imagerel $L$SEH_body_sgn0x_pty_mont_384x + DD imagerel $L$SEH_epilogue_sgn0x_pty_mont_384x + DD imagerel $L$SEH_info_sgn0x_pty_mont_384x_body + + DD imagerel $L$SEH_epilogue_sgn0x_pty_mont_384x + DD imagerel $L$SEH_end_sgn0x_pty_mont_384x + DD imagerel $L$SEH_info_sgn0x_pty_mont_384x_epilogue + + DD imagerel $L$SEH_begin_mulx_mont_384 + DD imagerel $L$SEH_body_mulx_mont_384 + DD imagerel $L$SEH_info_mulx_mont_384_prologue + + DD imagerel $L$SEH_body_mulx_mont_384 + DD imagerel $L$SEH_epilogue_mulx_mont_384 + DD imagerel $L$SEH_info_mulx_mont_384_body + + DD imagerel $L$SEH_epilogue_mulx_mont_384 + DD imagerel $L$SEH_end_mulx_mont_384 + DD imagerel $L$SEH_info_mulx_mont_384_epilogue + + DD imagerel $L$SEH_begin_sqrx_mont_384 + DD imagerel $L$SEH_body_sqrx_mont_384 + DD imagerel $L$SEH_info_sqrx_mont_384_prologue + + DD imagerel $L$SEH_body_sqrx_mont_384 + DD imagerel $L$SEH_epilogue_sqrx_mont_384 + DD imagerel $L$SEH_info_sqrx_mont_384_body + + DD imagerel $L$SEH_epilogue_sqrx_mont_384 + DD imagerel $L$SEH_end_sqrx_mont_384 + DD imagerel $L$SEH_info_sqrx_mont_384_epilogue + + DD imagerel $L$SEH_begin_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_body_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_384_prologue + + DD imagerel $L$SEH_body_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_epilogue_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_384_body + + DD imagerel $L$SEH_epilogue_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_end_sqrx_n_mul_mont_384 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_384_epilogue + + DD imagerel $L$SEH_begin_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_body_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_383_prologue + + DD imagerel $L$SEH_body_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_epilogue_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_383_body + + DD imagerel $L$SEH_epilogue_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_end_sqrx_n_mul_mont_383 + DD imagerel $L$SEH_info_sqrx_n_mul_mont_383_epilogue + + DD imagerel $L$SEH_begin_sqrx_mont_382x + DD imagerel $L$SEH_body_sqrx_mont_382x + DD imagerel $L$SEH_info_sqrx_mont_382x_prologue + + DD imagerel $L$SEH_body_sqrx_mont_382x + DD imagerel $L$SEH_epilogue_sqrx_mont_382x + DD imagerel $L$SEH_info_sqrx_mont_382x_body + + DD imagerel $L$SEH_epilogue_sqrx_mont_382x + DD imagerel $L$SEH_end_sqrx_mont_382x + DD imagerel $L$SEH_info_sqrx_mont_382x_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_mulx_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mulx_mont_384x_body:: +DB 1,0,18,0 +DB 000h,0f4h,029h,000h +DB 000h,0e4h,02ah,000h +DB 000h,0d4h,02bh,000h +DB 000h,0c4h,02ch,000h +DB 000h,034h,02dh,000h +DB 000h,054h,02eh,000h +DB 000h,074h,030h,000h +DB 000h,064h,031h,000h +DB 000h,001h,02fh,000h +$L$SEH_info_mulx_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_mont_384x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqrx_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mulx_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mulx_382x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_mulx_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_382x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqrx_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mulx_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mulx_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,000h,000h +DB 000h,0e4h,001h,000h +DB 000h,0d4h,002h,000h +DB 000h,0c4h,003h,000h +DB 000h,034h,004h,000h +DB 000h,054h,005h,000h +DB 000h,074h,007h,000h +DB 000h,064h,008h,000h +DB 000h,052h +DB 000h,000h +$L$SEH_info_mulx_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sqrx_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_redcx_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_redcx_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_redcx_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_fromx_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_fromx_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_fromx_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0x_pty_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0x_pty_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sgn0x_pty_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sgn0x_pty_mont_384x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sgn0x_pty_mont_384x_body:: +DB 1,0,17,0 +DB 000h,0f4h,001h,000h +DB 000h,0e4h,002h,000h +DB 000h,0d4h,003h,000h +DB 000h,0c4h,004h,000h +DB 000h,034h,005h,000h +DB 000h,054h,006h,000h +DB 000h,074h,008h,000h +DB 000h,064h,009h,000h +DB 000h,062h +DB 000h,000h +$L$SEH_info_sgn0x_pty_mont_384x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_mulx_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_mulx_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,003h,000h +DB 000h,0e4h,004h,000h +DB 000h,0d4h,005h,000h +DB 000h,0c4h,006h,000h +DB 000h,034h,007h,000h +DB 000h,054h,008h,000h +DB 000h,074h,00ah,000h +DB 000h,064h,00bh,000h +DB 000h,082h +DB 000h,000h +$L$SEH_info_mulx_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,003h,000h +DB 000h,0e4h,004h,000h +DB 000h,0d4h,005h,000h +DB 000h,0c4h,006h,000h +DB 000h,034h,007h,000h +DB 000h,054h,008h,000h +DB 000h,074h,00ah,000h +DB 000h,064h,00bh,000h +DB 000h,082h +DB 000h,000h +$L$SEH_info_sqrx_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_n_mul_mont_384_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_n_mul_mont_384_body:: +DB 1,0,17,0 +DB 000h,0f4h,005h,000h +DB 000h,0e4h,006h,000h +DB 000h,0d4h,007h,000h +DB 000h,0c4h,008h,000h +DB 000h,034h,009h,000h +DB 000h,054h,00ah,000h +DB 000h,074h,00ch,000h +DB 000h,064h,00dh,000h +DB 000h,0a2h +DB 000h,000h +$L$SEH_info_sqrx_n_mul_mont_384_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_n_mul_mont_383_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_n_mul_mont_383_body:: +DB 1,0,17,0 +DB 000h,0f4h,005h,000h +DB 000h,0e4h,006h,000h +DB 000h,0d4h,007h,000h +DB 000h,0c4h,008h,000h +DB 000h,034h,009h,000h +DB 000h,054h,00ah,000h +DB 000h,074h,00ch,000h +DB 000h,064h,00dh,000h +DB 000h,0a2h +DB 000h,000h +$L$SEH_info_sqrx_n_mul_mont_383_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + +$L$SEH_info_sqrx_mont_382x_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_sqrx_mont_382x_body:: +DB 1,0,18,0 +DB 000h,0f4h,011h,000h +DB 000h,0e4h,012h,000h +DB 000h,0d4h,013h,000h +DB 000h,0c4h,014h,000h +DB 000h,034h,015h,000h +DB 000h,054h,016h,000h +DB 000h,074h,018h,000h +DB 000h,064h,019h,000h +DB 000h,001h,017h,000h +$L$SEH_info_sqrx_mont_382x_epilogue:: +DB 1,0,4,0 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,000h,000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/build/win64/sha256-armv8.asm b/crypto/blst_src/build/win64/sha256-armv8.asm new file mode 100644 index 00000000000..0e0c54cb65b --- /dev/null +++ b/crypto/blst_src/build/win64/sha256-armv8.asm @@ -0,0 +1,1078 @@ +// +// Copyright Supranational LLC +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// ==================================================================== +// Written by Andy Polyakov, @dot-asm, initially for the OpenSSL +// project. +// ==================================================================== +// +// sha256_block procedure for ARMv8. +// +// This module is stripped of scalar code paths, with raionale that all +// known processors are NEON-capable. +// +// See original module at CRYPTOGAMS for further details. + + AREA |.text|,CODE,ALIGN=8,ARM64 + + ALIGN 64 + +|$LK256| + DCDU 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + DCDU 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + DCDU 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + DCDU 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + DCDU 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + DCDU 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + DCDU 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + DCDU 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + DCDU 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + DCDU 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + DCDU 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + DCDU 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + DCDU 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + DCDU 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + DCDU 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + DCDU 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + DCDU 0 //terminator + + DCB "SHA256 block transform for ARMv8, CRYPTOGAMS by @dot-asm",0 + ALIGN 4 + ALIGN 4 + + EXPORT |blst_sha256_block_armv8|[FUNC] + ALIGN 64 +|blst_sha256_block_armv8| PROC +|$Lv8_entry| + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ld1 {v0.4s,v1.4s},[x0] + adr x3,|$LK256| + +|$Loop_hw| + ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 + sub x2,x2,#1 + ld1 {v16.4s},[x3],#16 + rev32 v4.16b,v4.16b + rev32 v5.16b,v5.16b + rev32 v6.16b,v6.16b + rev32 v7.16b,v7.16b + orr v18.16b,v0.16b,v0.16b // offload + orr v19.16b,v1.16b,v1.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + DCDU 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + DCDU 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s + DCDU 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s + DCDU 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + DCDU 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + DCDU 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s + DCDU 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s + DCDU 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + DCDU 0x5e2828a4 //sha256su0 v4.16b,v5.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + DCDU 0x5e2828c5 //sha256su0 v5.16b,v6.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v6.4s + DCDU 0x5e2828e6 //sha256su0 v6.16b,v7.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + DCDU 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v7.4s + DCDU 0x5e282887 //sha256su0 v7.16b,v4.16b + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + DCDU 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b + ld1 {v17.4s},[x3],#16 + add v16.4s,v16.4s,v4.4s + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + ld1 {v16.4s},[x3],#16 + add v17.4s,v17.4s,v5.4s + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + ld1 {v17.4s},[x3] + add v16.4s,v16.4s,v6.4s + sub x3,x3,#64*4-16 // rewind + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s + DCDU 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s + + add v17.4s,v17.4s,v7.4s + orr v2.16b,v0.16b,v0.16b + DCDU 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s + DCDU 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s + + add v0.4s,v0.4s,v18.4s + add v1.4s,v1.4s,v19.4s + + cbnz x2,|$Loop_hw| + + st1 {v0.4s,v1.4s},[x0] + + ldr x29,[sp],#16 + ret + ENDP + + EXPORT |blst_sha256_block_data_order|[FUNC] + ALIGN 16 +|blst_sha256_block_data_order| PROC + stp x29, x30, [sp, #-16]! + mov x29, sp + sub sp,sp,#16*4 + + adr x16,|$LK256| + add x2,x1,x2,lsl#6 // len to point at the end of inp + + ld1 {v0.16b},[x1], #16 + ld1 {v1.16b},[x1], #16 + ld1 {v2.16b},[x1], #16 + ld1 {v3.16b},[x1], #16 + ld1 {v4.4s},[x16], #16 + ld1 {v5.4s},[x16], #16 + ld1 {v6.4s},[x16], #16 + ld1 {v7.4s},[x16], #16 + rev32 v0.16b,v0.16b // yes, even on + rev32 v1.16b,v1.16b // big-endian + rev32 v2.16b,v2.16b + rev32 v3.16b,v3.16b + mov x17,sp + add v4.4s,v4.4s,v0.4s + add v5.4s,v5.4s,v1.4s + add v6.4s,v6.4s,v2.4s + st1 {v4.4s,v5.4s},[x17], #32 + add v7.4s,v7.4s,v3.4s + st1 {v6.4s,v7.4s},[x17] + sub x17,x17,#32 + + ldp w3,w4,[x0] + ldp w5,w6,[x0,#8] + ldp w7,w8,[x0,#16] + ldp w9,w10,[x0,#24] + ldr w12,[sp,#0] + mov w13,wzr + eor w14,w4,w5 + mov w15,wzr + b |$L_00_48| + + ALIGN 16 +|$L_00_48| + ext8 v4.16b,v0.16b,v1.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext8 v7.16b,v2.16b,v3.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v3.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v0.4s,v0.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v0.4s,v0.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v0.4s,v0.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v0.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v0.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v0.4s,#15 + add w8,w8,w12 + ushr v17.4s,v0.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v0.4s,#13 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v0.4s,v0.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v0.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext8 v4.16b,v1.16b,v2.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext8 v7.16b,v3.16b,v0.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v0.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v1.4s,v1.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v1.4s,v1.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v1.4s,v1.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v1.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v1.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v1.4s,#15 + add w4,w4,w12 + ushr v17.4s,v1.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v1.4s,#13 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v1.4s,v1.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v1.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + ext8 v4.16b,v2.16b,v3.16b,#4 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + bic w15,w9,w7 + ext8 v7.16b,v0.16b,v1.16b,#4 + eor w11,w7,w7,ror#5 + add w3,w3,w13 + mov d19,v1.d[1] + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w3,w3,ror#11 + ushr v5.4s,v4.4s,#3 + add w10,w10,w12 + add v2.4s,v2.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + ushr v7.4s,v4.4s,#18 + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w6,w6,w10 + sli v7.4s,v4.4s,#14 + eor w14,w14,w4 + ushr v16.4s,v19.4s,#17 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + eor v5.16b,v5.16b,v7.16b + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + sli v16.4s,v19.4s,#15 + add w10,w10,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + ushr v7.4s,v19.4s,#19 + add w9,w9,w12 + ror w11,w11,#6 + add v2.4s,v2.4s,v5.4s + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + sli v7.4s,v19.4s,#13 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + eor v17.16b,v17.16b,v7.16b + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + add v2.4s,v2.4s,v17.4s + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + ushr v18.4s,v2.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v2.4s,#10 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + sli v18.4s,v2.4s,#15 + add w8,w8,w12 + ushr v17.4s,v2.4s,#19 + ror w11,w11,#6 + eor w13,w9,w10 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w9,ror#20 + add w8,w8,w11 + sli v17.4s,v2.4s,#13 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w4,w4,w8 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w10 + eor v17.16b,v17.16b,v17.16b + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + mov v17.d[1],v19.d[0] + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + add v2.4s,v2.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add v4.4s,v4.4s,v2.4s + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + ext8 v4.16b,v3.16b,v0.16b,#4 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + bic w15,w5,w3 + ext8 v7.16b,v1.16b,v2.16b,#4 + eor w11,w3,w3,ror#5 + add w7,w7,w13 + mov d19,v2.d[1] + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + ushr v6.4s,v4.4s,#7 + eor w15,w7,w7,ror#11 + ushr v5.4s,v4.4s,#3 + add w6,w6,w12 + add v3.4s,v3.4s,v7.4s + ror w11,w11,#6 + sli v6.4s,v4.4s,#25 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + ushr v7.4s,v4.4s,#18 + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + eor v5.16b,v5.16b,v6.16b + ror w15,w15,#2 + add w10,w10,w6 + sli v7.4s,v4.4s,#14 + eor w14,w14,w8 + ushr v16.4s,v19.4s,#17 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + eor v5.16b,v5.16b,v7.16b + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + sli v16.4s,v19.4s,#15 + add w6,w6,w14 + orr w12,w12,w15 + ushr v17.4s,v19.4s,#10 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + ushr v7.4s,v19.4s,#19 + add w5,w5,w12 + ror w11,w11,#6 + add v3.4s,v3.4s,v5.4s + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + sli v7.4s,v19.4s,#13 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + eor v17.16b,v17.16b,v16.16b + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + eor v17.16b,v17.16b,v7.16b + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + add v3.4s,v3.4s,v17.4s + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + ushr v18.4s,v3.4s,#17 + orr w12,w12,w15 + ushr v19.4s,v3.4s,#10 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + sli v18.4s,v3.4s,#15 + add w4,w4,w12 + ushr v17.4s,v3.4s,#19 + ror w11,w11,#6 + eor w13,w5,w6 + eor v19.16b,v19.16b,v18.16b + eor w15,w15,w5,ror#20 + add w4,w4,w11 + sli v17.4s,v3.4s,#13 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + ld1 {v4.4s},[x16], #16 + add w8,w8,w4 + eor v19.16b,v19.16b,v17.16b + eor w14,w14,w6 + eor v17.16b,v17.16b,v17.16b + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + mov v17.d[1],v19.d[0] + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + add v3.4s,v3.4s,v17.4s + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add v4.4s,v4.4s,v3.4s + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[x16] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + cmp w12,#0 // check for K256 terminator + ldr w12,[sp,#0] + sub x17,x17,#64 + bne |$L_00_48| + + sub x16,x16,#256 // rewind x16 + cmp x1,x2 + mov x17, #64 + cseleq x17,x17,xzr + sub x1,x1,x17 // avoid SEGV + mov x17,sp + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v0.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v0.16b,v0.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v0.4s + add w10,w10,w11 + ldr w12,[sp,#4] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#8] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#12] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#16] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v1.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v1.16b,v1.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v1.4s + add w6,w6,w11 + ldr w12,[sp,#20] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#24] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#28] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + ldr w12,[sp,#32] + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w10,w10,w12 + add w3,w3,w15 + and w12,w8,w7 + ld1 {v2.16b},[x1],#16 + bic w15,w9,w7 + eor w11,w7,w7,ror#5 + ld1 {v4.4s},[x16],#16 + add w3,w3,w13 + orr w12,w12,w15 + eor w11,w11,w7,ror#19 + eor w15,w3,w3,ror#11 + rev32 v2.16b,v2.16b + add w10,w10,w12 + ror w11,w11,#6 + eor w13,w3,w4 + eor w15,w15,w3,ror#20 + add v4.4s,v4.4s,v2.4s + add w10,w10,w11 + ldr w12,[sp,#36] + and w14,w14,w13 + ror w15,w15,#2 + add w6,w6,w10 + eor w14,w14,w4 + add w9,w9,w12 + add w10,w10,w15 + and w12,w7,w6 + bic w15,w8,w6 + eor w11,w6,w6,ror#5 + add w10,w10,w14 + orr w12,w12,w15 + eor w11,w11,w6,ror#19 + eor w15,w10,w10,ror#11 + add w9,w9,w12 + ror w11,w11,#6 + eor w14,w10,w3 + eor w15,w15,w10,ror#20 + add w9,w9,w11 + ldr w12,[sp,#40] + and w13,w13,w14 + ror w15,w15,#2 + add w5,w5,w9 + eor w13,w13,w3 + add w8,w8,w12 + add w9,w9,w15 + and w12,w6,w5 + bic w15,w7,w5 + eor w11,w5,w5,ror#5 + add w9,w9,w13 + orr w12,w12,w15 + eor w11,w11,w5,ror#19 + eor w15,w9,w9,ror#11 + add w8,w8,w12 + ror w11,w11,#6 + eor w13,w9,w10 + eor w15,w15,w9,ror#20 + add w8,w8,w11 + ldr w12,[sp,#44] + and w14,w14,w13 + ror w15,w15,#2 + add w4,w4,w8 + eor w14,w14,w10 + add w7,w7,w12 + add w8,w8,w15 + and w12,w5,w4 + bic w15,w6,w4 + eor w11,w4,w4,ror#5 + add w8,w8,w14 + orr w12,w12,w15 + eor w11,w11,w4,ror#19 + eor w15,w8,w8,ror#11 + add w7,w7,w12 + ror w11,w11,#6 + eor w14,w8,w9 + eor w15,w15,w8,ror#20 + add w7,w7,w11 + ldr w12,[sp,#48] + and w13,w13,w14 + ror w15,w15,#2 + add w3,w3,w7 + eor w13,w13,w9 + st1 {v4.4s},[x17], #16 + add w6,w6,w12 + add w7,w7,w15 + and w12,w4,w3 + ld1 {v3.16b},[x1],#16 + bic w15,w5,w3 + eor w11,w3,w3,ror#5 + ld1 {v4.4s},[x16],#16 + add w7,w7,w13 + orr w12,w12,w15 + eor w11,w11,w3,ror#19 + eor w15,w7,w7,ror#11 + rev32 v3.16b,v3.16b + add w6,w6,w12 + ror w11,w11,#6 + eor w13,w7,w8 + eor w15,w15,w7,ror#20 + add v4.4s,v4.4s,v3.4s + add w6,w6,w11 + ldr w12,[sp,#52] + and w14,w14,w13 + ror w15,w15,#2 + add w10,w10,w6 + eor w14,w14,w8 + add w5,w5,w12 + add w6,w6,w15 + and w12,w3,w10 + bic w15,w4,w10 + eor w11,w10,w10,ror#5 + add w6,w6,w14 + orr w12,w12,w15 + eor w11,w11,w10,ror#19 + eor w15,w6,w6,ror#11 + add w5,w5,w12 + ror w11,w11,#6 + eor w14,w6,w7 + eor w15,w15,w6,ror#20 + add w5,w5,w11 + ldr w12,[sp,#56] + and w13,w13,w14 + ror w15,w15,#2 + add w9,w9,w5 + eor w13,w13,w7 + add w4,w4,w12 + add w5,w5,w15 + and w12,w10,w9 + bic w15,w3,w9 + eor w11,w9,w9,ror#5 + add w5,w5,w13 + orr w12,w12,w15 + eor w11,w11,w9,ror#19 + eor w15,w5,w5,ror#11 + add w4,w4,w12 + ror w11,w11,#6 + eor w13,w5,w6 + eor w15,w15,w5,ror#20 + add w4,w4,w11 + ldr w12,[sp,#60] + and w14,w14,w13 + ror w15,w15,#2 + add w8,w8,w4 + eor w14,w14,w6 + add w3,w3,w12 + add w4,w4,w15 + and w12,w9,w8 + bic w15,w10,w8 + eor w11,w8,w8,ror#5 + add w4,w4,w14 + orr w12,w12,w15 + eor w11,w11,w8,ror#19 + eor w15,w4,w4,ror#11 + add w3,w3,w12 + ror w11,w11,#6 + eor w14,w4,w5 + eor w15,w15,w4,ror#20 + add w3,w3,w11 + and w13,w13,w14 + ror w15,w15,#2 + add w7,w7,w3 + eor w13,w13,w5 + st1 {v4.4s},[x17], #16 + add w3,w3,w15 // h+=Sigma0(a) from the past + ldp w11,w12,[x0,#0] + add w3,w3,w13 // h+=Maj(a,b,c) from the past + ldp w13,w14,[x0,#8] + add w3,w3,w11 // accumulate + add w4,w4,w12 + ldp w11,w12,[x0,#16] + add w5,w5,w13 + add w6,w6,w14 + ldp w13,w14,[x0,#24] + add w7,w7,w11 + add w8,w8,w12 + ldr w12,[sp,#0] + stp w3,w4,[x0,#0] + add w9,w9,w13 + mov w13,wzr + stp w5,w6,[x0,#8] + add w10,w10,w14 + stp w7,w8,[x0,#16] + eor w14,w4,w5 + stp w9,w10,[x0,#24] + mov w15,wzr + mov x17,sp + bne |$L_00_48| + + ldr x29,[x29] + add sp,sp,#16*4+16 + ret + ENDP + + + EXPORT |blst_sha256_emit|[FUNC] + ALIGN 16 +|blst_sha256_emit| PROC + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] +#ifndef __AARCH64EB__ + rev x4,x4 + rev x5,x5 + rev x6,x6 + rev x7,x7 +#endif + str w4,[x0,#4] + lsr x4,x4,#32 + str w5,[x0,#12] + lsr x5,x5,#32 + str w6,[x0,#20] + lsr x6,x6,#32 + str w7,[x0,#28] + lsr x7,x7,#32 + str w4,[x0,#0] + str w5,[x0,#8] + str w6,[x0,#16] + str w7,[x0,#24] + ret + ENDP + + + + EXPORT |blst_sha256_bcopy|[FUNC] + ALIGN 16 +|blst_sha256_bcopy| PROC +|$Loop_bcopy| + ldrb w3,[x1],#1 + sub x2,x2,#1 + strb w3,[x0],#1 + cbnz x2,|$Loop_bcopy| + ret + ENDP + + + + EXPORT |blst_sha256_hcopy|[FUNC] + ALIGN 16 +|blst_sha256_hcopy| PROC + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + stp x4,x5,[x0] + stp x6,x7,[x0,#16] + ret + ENDP + END diff --git a/crypto/blst_src/build/win64/sha256-x86_64.asm b/crypto/blst_src/build/win64/sha256-x86_64.asm new file mode 100644 index 00000000000..d3b409235e7 --- /dev/null +++ b/crypto/blst_src/build/win64/sha256-x86_64.asm @@ -0,0 +1,1570 @@ +OPTION DOTNAME +.text$ SEGMENT ALIGN(256) 'CODE' + +ALIGN 64 + +K256:: + DD 0428a2f98h,071374491h,0b5c0fbcfh,0e9b5dba5h + DD 03956c25bh,059f111f1h,0923f82a4h,0ab1c5ed5h + DD 0d807aa98h,012835b01h,0243185beh,0550c7dc3h + DD 072be5d74h,080deb1feh,09bdc06a7h,0c19bf174h + DD 0e49b69c1h,0efbe4786h,00fc19dc6h,0240ca1cch + DD 02de92c6fh,04a7484aah,05cb0a9dch,076f988dah + DD 0983e5152h,0a831c66dh,0b00327c8h,0bf597fc7h + DD 0c6e00bf3h,0d5a79147h,006ca6351h,014292967h + DD 027b70a85h,02e1b2138h,04d2c6dfch,053380d13h + DD 0650a7354h,0766a0abbh,081c2c92eh,092722c85h + DD 0a2bfe8a1h,0a81a664bh,0c24b8b70h,0c76c51a3h + DD 0d192e819h,0d6990624h,0f40e3585h,0106aa070h + DD 019a4c116h,01e376c08h,02748774ch,034b0bcb5h + DD 0391c0cb3h,04ed8aa4ah,05b9cca4fh,0682e6ff3h + DD 0748f82eeh,078a5636fh,084c87814h,08cc70208h + DD 090befffah,0a4506cebh,0bef9a3f7h,0c67178f2h + + DD 000010203h,004050607h,008090a0bh,00c0d0e0fh + DD 003020100h,00b0a0908h,0ffffffffh,0ffffffffh + DD 0ffffffffh,0ffffffffh,003020100h,00b0a0908h +DB 83,72,65,50,53,54,32,98,108,111,99,107,32,116,114,97 +DB 110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54 +DB 52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121 +DB 32,64,100,111,116,45,97,115,109,0 +PUBLIC blst_sha256_block_data_order_shaext + + +ALIGN 64 +blst_sha256_block_data_order_shaext PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_blst_sha256_block_data_order_shaext:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + sub rsp,058h + + movaps XMMWORD PTR[(-88)+r11],xmm6 + + movaps XMMWORD PTR[(-72)+r11],xmm7 + + movaps XMMWORD PTR[(-56)+r11],xmm8 + + movaps XMMWORD PTR[(-40)+r11],xmm9 + + movaps XMMWORD PTR[(-24)+r11],xmm10 + +$L$SEH_body_blst_sha256_block_data_order_shaext:: + + lea rcx,QWORD PTR[((K256+128))] + movdqu xmm1,XMMWORD PTR[rdi] + movdqu xmm2,XMMWORD PTR[16+rdi] + movdqa xmm7,XMMWORD PTR[((256-128))+rcx] + + pshufd xmm0,xmm1,01bh + pshufd xmm1,xmm1,0b1h + pshufd xmm2,xmm2,01bh + movdqa xmm8,xmm7 +DB 102,15,58,15,202,8 + punpcklqdq xmm2,xmm0 + jmp $L$oop_shaext + +ALIGN 16 +$L$oop_shaext:: + movdqu xmm3,XMMWORD PTR[rsi] + movdqu xmm4,XMMWORD PTR[16+rsi] + movdqu xmm5,XMMWORD PTR[32+rsi] +DB 102,15,56,0,223 + movdqu xmm6,XMMWORD PTR[48+rsi] + + movdqa xmm0,XMMWORD PTR[((0-128))+rcx] + paddd xmm0,xmm3 +DB 102,15,56,0,231 + movdqa xmm10,xmm2 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + nop + movdqa xmm9,xmm1 +DB 15,56,203,202 + + movdqa xmm0,XMMWORD PTR[((16-128))+rcx] + paddd xmm0,xmm4 +DB 102,15,56,0,239 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + lea rsi,QWORD PTR[64+rsi] +DB 15,56,204,220 +DB 15,56,203,202 + + movdqa xmm0,XMMWORD PTR[((32-128))+rcx] + paddd xmm0,xmm5 +DB 102,15,56,0,247 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm6 +DB 102,15,58,15,253,4 + nop + paddd xmm3,xmm7 +DB 15,56,204,229 +DB 15,56,203,202 + + movdqa xmm0,XMMWORD PTR[((48-128))+rcx] + paddd xmm0,xmm6 +DB 15,56,205,222 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm3 +DB 102,15,58,15,254,4 + nop + paddd xmm4,xmm7 +DB 15,56,204,238 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((64-128))+rcx] + paddd xmm0,xmm3 +DB 15,56,205,227 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm4 +DB 102,15,58,15,251,4 + nop + paddd xmm5,xmm7 +DB 15,56,204,243 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((80-128))+rcx] + paddd xmm0,xmm4 +DB 15,56,205,236 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm5 +DB 102,15,58,15,252,4 + nop + paddd xmm6,xmm7 +DB 15,56,204,220 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((96-128))+rcx] + paddd xmm0,xmm5 +DB 15,56,205,245 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm6 +DB 102,15,58,15,253,4 + nop + paddd xmm3,xmm7 +DB 15,56,204,229 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((112-128))+rcx] + paddd xmm0,xmm6 +DB 15,56,205,222 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm3 +DB 102,15,58,15,254,4 + nop + paddd xmm4,xmm7 +DB 15,56,204,238 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((128-128))+rcx] + paddd xmm0,xmm3 +DB 15,56,205,227 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm4 +DB 102,15,58,15,251,4 + nop + paddd xmm5,xmm7 +DB 15,56,204,243 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((144-128))+rcx] + paddd xmm0,xmm4 +DB 15,56,205,236 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm5 +DB 102,15,58,15,252,4 + nop + paddd xmm6,xmm7 +DB 15,56,204,220 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((160-128))+rcx] + paddd xmm0,xmm5 +DB 15,56,205,245 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm6 +DB 102,15,58,15,253,4 + nop + paddd xmm3,xmm7 +DB 15,56,204,229 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((176-128))+rcx] + paddd xmm0,xmm6 +DB 15,56,205,222 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm3 +DB 102,15,58,15,254,4 + nop + paddd xmm4,xmm7 +DB 15,56,204,238 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((192-128))+rcx] + paddd xmm0,xmm3 +DB 15,56,205,227 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm4 +DB 102,15,58,15,251,4 + nop + paddd xmm5,xmm7 +DB 15,56,204,243 +DB 15,56,203,202 + movdqa xmm0,XMMWORD PTR[((208-128))+rcx] + paddd xmm0,xmm4 +DB 15,56,205,236 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + movdqa xmm7,xmm5 +DB 102,15,58,15,252,4 +DB 15,56,203,202 + paddd xmm6,xmm7 + + movdqa xmm0,XMMWORD PTR[((224-128))+rcx] + paddd xmm0,xmm5 +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh +DB 15,56,205,245 + movdqa xmm7,xmm8 +DB 15,56,203,202 + + movdqa xmm0,XMMWORD PTR[((240-128))+rcx] + paddd xmm0,xmm6 + nop +DB 15,56,203,209 + pshufd xmm0,xmm0,00eh + dec rdx + nop +DB 15,56,203,202 + + paddd xmm2,xmm10 + paddd xmm1,xmm9 + jnz $L$oop_shaext + + pshufd xmm2,xmm2,0b1h + pshufd xmm7,xmm1,01bh + pshufd xmm1,xmm1,0b1h + punpckhqdq xmm1,xmm2 +DB 102,15,58,15,215,8 + + movdqu XMMWORD PTR[rdi],xmm1 + movdqu XMMWORD PTR[16+rdi],xmm2 + movaps xmm6,XMMWORD PTR[((-88))+r11] + movaps xmm7,XMMWORD PTR[((-72))+r11] + movaps xmm8,XMMWORD PTR[((-56))+r11] + movaps xmm9,XMMWORD PTR[((-40))+r11] + movaps xmm10,XMMWORD PTR[((-24))+r11] + mov rsp,r11 + +$L$SEH_epilogue_blst_sha256_block_data_order_shaext:: + mov rdi,QWORD PTR[8+r11] ;WIN64 epilogue + mov rsi,QWORD PTR[16+r11] + + DB 0F3h,0C3h ;repret + +$L$SEH_end_blst_sha256_block_data_order_shaext:: +blst_sha256_block_data_order_shaext ENDP +PUBLIC blst_sha256_block_data_order + + +ALIGN 64 +blst_sha256_block_data_order PROC PUBLIC + DB 243,15,30,250 + mov QWORD PTR[8+rsp],rdi ;WIN64 prologue + mov QWORD PTR[16+rsp],rsi + mov r11,rsp +$L$SEH_begin_blst_sha256_block_data_order:: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + shl rdx,4 + sub rsp,104 + + lea rdx,QWORD PTR[rdx*4+rsi] + mov QWORD PTR[rsp],rdi + + mov QWORD PTR[16+rsp],rdx + movaps XMMWORD PTR[32+rsp],xmm6 + + movaps XMMWORD PTR[48+rsp],xmm7 + + movaps XMMWORD PTR[64+rsp],xmm8 + + movaps XMMWORD PTR[80+rsp],xmm9 + + mov rbp,rsp + +$L$SEH_body_blst_sha256_block_data_order:: + + + lea rsp,QWORD PTR[((-64))+rsp] + mov eax,DWORD PTR[rdi] + and rsp,-64 + mov ebx,DWORD PTR[4+rdi] + mov ecx,DWORD PTR[8+rdi] + mov edx,DWORD PTR[12+rdi] + mov r8d,DWORD PTR[16+rdi] + mov r9d,DWORD PTR[20+rdi] + mov r10d,DWORD PTR[24+rdi] + mov r11d,DWORD PTR[28+rdi] + + + jmp $L$loop_ssse3 +ALIGN 16 +$L$loop_ssse3:: + movdqa xmm7,XMMWORD PTR[((K256+256))] + mov QWORD PTR[8+rbp],rsi + movdqu xmm0,XMMWORD PTR[rsi] + movdqu xmm1,XMMWORD PTR[16+rsi] + movdqu xmm2,XMMWORD PTR[32+rsi] +DB 102,15,56,0,199 + movdqu xmm3,XMMWORD PTR[48+rsi] + lea rsi,QWORD PTR[K256] +DB 102,15,56,0,207 + movdqa xmm4,XMMWORD PTR[rsi] + movdqa xmm5,XMMWORD PTR[16+rsi] +DB 102,15,56,0,215 + paddd xmm4,xmm0 + movdqa xmm6,XMMWORD PTR[32+rsi] +DB 102,15,56,0,223 + movdqa xmm7,XMMWORD PTR[48+rsi] + paddd xmm5,xmm1 + paddd xmm6,xmm2 + paddd xmm7,xmm3 + movdqa XMMWORD PTR[rsp],xmm4 + mov r14d,eax + movdqa XMMWORD PTR[16+rsp],xmm5 + mov edi,ebx + movdqa XMMWORD PTR[32+rsp],xmm6 + xor edi,ecx + movdqa XMMWORD PTR[48+rsp],xmm7 + mov r13d,r8d + jmp $L$ssse3_00_47 + +ALIGN 16 +$L$ssse3_00_47:: + sub rsi,-64 + ror r13d,14 + movdqa xmm4,xmm1 + mov eax,r14d + mov r12d,r9d + movdqa xmm7,xmm3 + ror r14d,9 + xor r13d,r8d + xor r12d,r10d + ror r13d,5 + xor r14d,eax +DB 102,15,58,15,224,4 + and r12d,r8d + xor r13d,r8d +DB 102,15,58,15,250,4 + add r11d,DWORD PTR[rsp] + mov r15d,eax + xor r12d,r10d + ror r14d,11 + movdqa xmm5,xmm4 + xor r15d,ebx + add r11d,r12d + movdqa xmm6,xmm4 + ror r13d,6 + and edi,r15d + psrld xmm4,3 + xor r14d,eax + add r11d,r13d + xor edi,ebx + paddd xmm0,xmm7 + ror r14d,2 + add edx,r11d + psrld xmm6,7 + add r11d,edi + mov r13d,edx + pshufd xmm7,xmm3,250 + add r14d,r11d + ror r13d,14 + pslld xmm5,14 + mov r11d,r14d + mov r12d,r8d + pxor xmm4,xmm6 + ror r14d,9 + xor r13d,edx + xor r12d,r9d + ror r13d,5 + psrld xmm6,11 + xor r14d,r11d + pxor xmm4,xmm5 + and r12d,edx + xor r13d,edx + pslld xmm5,11 + add r10d,DWORD PTR[4+rsp] + mov edi,r11d + pxor xmm4,xmm6 + xor r12d,r9d + ror r14d,11 + movdqa xmm6,xmm7 + xor edi,eax + add r10d,r12d + pxor xmm4,xmm5 + ror r13d,6 + and r15d,edi + xor r14d,r11d + psrld xmm7,10 + add r10d,r13d + xor r15d,eax + paddd xmm0,xmm4 + ror r14d,2 + add ecx,r10d + psrlq xmm6,17 + add r10d,r15d + mov r13d,ecx + add r14d,r10d + pxor xmm7,xmm6 + ror r13d,14 + mov r10d,r14d + mov r12d,edx + ror r14d,9 + psrlq xmm6,2 + xor r13d,ecx + xor r12d,r8d + pxor xmm7,xmm6 + ror r13d,5 + xor r14d,r10d + and r12d,ecx + pshufd xmm7,xmm7,128 + xor r13d,ecx + add r9d,DWORD PTR[8+rsp] + mov r15d,r10d + psrldq xmm7,8 + xor r12d,r8d + ror r14d,11 + xor r15d,r11d + add r9d,r12d + ror r13d,6 + paddd xmm0,xmm7 + and edi,r15d + xor r14d,r10d + add r9d,r13d + pshufd xmm7,xmm0,80 + xor edi,r11d + ror r14d,2 + add ebx,r9d + movdqa xmm6,xmm7 + add r9d,edi + mov r13d,ebx + psrld xmm7,10 + add r14d,r9d + ror r13d,14 + psrlq xmm6,17 + mov r9d,r14d + mov r12d,ecx + pxor xmm7,xmm6 + ror r14d,9 + xor r13d,ebx + xor r12d,edx + ror r13d,5 + xor r14d,r9d + psrlq xmm6,2 + and r12d,ebx + xor r13d,ebx + add r8d,DWORD PTR[12+rsp] + pxor xmm7,xmm6 + mov edi,r9d + xor r12d,edx + ror r14d,11 + pshufd xmm7,xmm7,8 + xor edi,r10d + add r8d,r12d + movdqa xmm6,XMMWORD PTR[rsi] + ror r13d,6 + and r15d,edi + pslldq xmm7,8 + xor r14d,r9d + add r8d,r13d + xor r15d,r10d + paddd xmm0,xmm7 + ror r14d,2 + add eax,r8d + add r8d,r15d + paddd xmm6,xmm0 + mov r13d,eax + add r14d,r8d + movdqa XMMWORD PTR[rsp],xmm6 + ror r13d,14 + movdqa xmm4,xmm2 + mov r8d,r14d + mov r12d,ebx + movdqa xmm7,xmm0 + ror r14d,9 + xor r13d,eax + xor r12d,ecx + ror r13d,5 + xor r14d,r8d +DB 102,15,58,15,225,4 + and r12d,eax + xor r13d,eax +DB 102,15,58,15,251,4 + add edx,DWORD PTR[16+rsp] + mov r15d,r8d + xor r12d,ecx + ror r14d,11 + movdqa xmm5,xmm4 + xor r15d,r9d + add edx,r12d + movdqa xmm6,xmm4 + ror r13d,6 + and edi,r15d + psrld xmm4,3 + xor r14d,r8d + add edx,r13d + xor edi,r9d + paddd xmm1,xmm7 + ror r14d,2 + add r11d,edx + psrld xmm6,7 + add edx,edi + mov r13d,r11d + pshufd xmm7,xmm0,250 + add r14d,edx + ror r13d,14 + pslld xmm5,14 + mov edx,r14d + mov r12d,eax + pxor xmm4,xmm6 + ror r14d,9 + xor r13d,r11d + xor r12d,ebx + ror r13d,5 + psrld xmm6,11 + xor r14d,edx + pxor xmm4,xmm5 + and r12d,r11d + xor r13d,r11d + pslld xmm5,11 + add ecx,DWORD PTR[20+rsp] + mov edi,edx + pxor xmm4,xmm6 + xor r12d,ebx + ror r14d,11 + movdqa xmm6,xmm7 + xor edi,r8d + add ecx,r12d + pxor xmm4,xmm5 + ror r13d,6 + and r15d,edi + xor r14d,edx + psrld xmm7,10 + add ecx,r13d + xor r15d,r8d + paddd xmm1,xmm4 + ror r14d,2 + add r10d,ecx + psrlq xmm6,17 + add ecx,r15d + mov r13d,r10d + add r14d,ecx + pxor xmm7,xmm6 + ror r13d,14 + mov ecx,r14d + mov r12d,r11d + ror r14d,9 + psrlq xmm6,2 + xor r13d,r10d + xor r12d,eax + pxor xmm7,xmm6 + ror r13d,5 + xor r14d,ecx + and r12d,r10d + pshufd xmm7,xmm7,128 + xor r13d,r10d + add ebx,DWORD PTR[24+rsp] + mov r15d,ecx + psrldq xmm7,8 + xor r12d,eax + ror r14d,11 + xor r15d,edx + add ebx,r12d + ror r13d,6 + paddd xmm1,xmm7 + and edi,r15d + xor r14d,ecx + add ebx,r13d + pshufd xmm7,xmm1,80 + xor edi,edx + ror r14d,2 + add r9d,ebx + movdqa xmm6,xmm7 + add ebx,edi + mov r13d,r9d + psrld xmm7,10 + add r14d,ebx + ror r13d,14 + psrlq xmm6,17 + mov ebx,r14d + mov r12d,r10d + pxor xmm7,xmm6 + ror r14d,9 + xor r13d,r9d + xor r12d,r11d + ror r13d,5 + xor r14d,ebx + psrlq xmm6,2 + and r12d,r9d + xor r13d,r9d + add eax,DWORD PTR[28+rsp] + pxor xmm7,xmm6 + mov edi,ebx + xor r12d,r11d + ror r14d,11 + pshufd xmm7,xmm7,8 + xor edi,ecx + add eax,r12d + movdqa xmm6,XMMWORD PTR[16+rsi] + ror r13d,6 + and r15d,edi + pslldq xmm7,8 + xor r14d,ebx + add eax,r13d + xor r15d,ecx + paddd xmm1,xmm7 + ror r14d,2 + add r8d,eax + add eax,r15d + paddd xmm6,xmm1 + mov r13d,r8d + add r14d,eax + movdqa XMMWORD PTR[16+rsp],xmm6 + ror r13d,14 + movdqa xmm4,xmm3 + mov eax,r14d + mov r12d,r9d + movdqa xmm7,xmm1 + ror r14d,9 + xor r13d,r8d + xor r12d,r10d + ror r13d,5 + xor r14d,eax +DB 102,15,58,15,226,4 + and r12d,r8d + xor r13d,r8d +DB 102,15,58,15,248,4 + add r11d,DWORD PTR[32+rsp] + mov r15d,eax + xor r12d,r10d + ror r14d,11 + movdqa xmm5,xmm4 + xor r15d,ebx + add r11d,r12d + movdqa xmm6,xmm4 + ror r13d,6 + and edi,r15d + psrld xmm4,3 + xor r14d,eax + add r11d,r13d + xor edi,ebx + paddd xmm2,xmm7 + ror r14d,2 + add edx,r11d + psrld xmm6,7 + add r11d,edi + mov r13d,edx + pshufd xmm7,xmm1,250 + add r14d,r11d + ror r13d,14 + pslld xmm5,14 + mov r11d,r14d + mov r12d,r8d + pxor xmm4,xmm6 + ror r14d,9 + xor r13d,edx + xor r12d,r9d + ror r13d,5 + psrld xmm6,11 + xor r14d,r11d + pxor xmm4,xmm5 + and r12d,edx + xor r13d,edx + pslld xmm5,11 + add r10d,DWORD PTR[36+rsp] + mov edi,r11d + pxor xmm4,xmm6 + xor r12d,r9d + ror r14d,11 + movdqa xmm6,xmm7 + xor edi,eax + add r10d,r12d + pxor xmm4,xmm5 + ror r13d,6 + and r15d,edi + xor r14d,r11d + psrld xmm7,10 + add r10d,r13d + xor r15d,eax + paddd xmm2,xmm4 + ror r14d,2 + add ecx,r10d + psrlq xmm6,17 + add r10d,r15d + mov r13d,ecx + add r14d,r10d + pxor xmm7,xmm6 + ror r13d,14 + mov r10d,r14d + mov r12d,edx + ror r14d,9 + psrlq xmm6,2 + xor r13d,ecx + xor r12d,r8d + pxor xmm7,xmm6 + ror r13d,5 + xor r14d,r10d + and r12d,ecx + pshufd xmm7,xmm7,128 + xor r13d,ecx + add r9d,DWORD PTR[40+rsp] + mov r15d,r10d + psrldq xmm7,8 + xor r12d,r8d + ror r14d,11 + xor r15d,r11d + add r9d,r12d + ror r13d,6 + paddd xmm2,xmm7 + and edi,r15d + xor r14d,r10d + add r9d,r13d + pshufd xmm7,xmm2,80 + xor edi,r11d + ror r14d,2 + add ebx,r9d + movdqa xmm6,xmm7 + add r9d,edi + mov r13d,ebx + psrld xmm7,10 + add r14d,r9d + ror r13d,14 + psrlq xmm6,17 + mov r9d,r14d + mov r12d,ecx + pxor xmm7,xmm6 + ror r14d,9 + xor r13d,ebx + xor r12d,edx + ror r13d,5 + xor r14d,r9d + psrlq xmm6,2 + and r12d,ebx + xor r13d,ebx + add r8d,DWORD PTR[44+rsp] + pxor xmm7,xmm6 + mov edi,r9d + xor r12d,edx + ror r14d,11 + pshufd xmm7,xmm7,8 + xor edi,r10d + add r8d,r12d + movdqa xmm6,XMMWORD PTR[32+rsi] + ror r13d,6 + and r15d,edi + pslldq xmm7,8 + xor r14d,r9d + add r8d,r13d + xor r15d,r10d + paddd xmm2,xmm7 + ror r14d,2 + add eax,r8d + add r8d,r15d + paddd xmm6,xmm2 + mov r13d,eax + add r14d,r8d + movdqa XMMWORD PTR[32+rsp],xmm6 + ror r13d,14 + movdqa xmm4,xmm0 + mov r8d,r14d + mov r12d,ebx + movdqa xmm7,xmm2 + ror r14d,9 + xor r13d,eax + xor r12d,ecx + ror r13d,5 + xor r14d,r8d +DB 102,15,58,15,227,4 + and r12d,eax + xor r13d,eax +DB 102,15,58,15,249,4 + add edx,DWORD PTR[48+rsp] + mov r15d,r8d + xor r12d,ecx + ror r14d,11 + movdqa xmm5,xmm4 + xor r15d,r9d + add edx,r12d + movdqa xmm6,xmm4 + ror r13d,6 + and edi,r15d + psrld xmm4,3 + xor r14d,r8d + add edx,r13d + xor edi,r9d + paddd xmm3,xmm7 + ror r14d,2 + add r11d,edx + psrld xmm6,7 + add edx,edi + mov r13d,r11d + pshufd xmm7,xmm2,250 + add r14d,edx + ror r13d,14 + pslld xmm5,14 + mov edx,r14d + mov r12d,eax + pxor xmm4,xmm6 + ror r14d,9 + xor r13d,r11d + xor r12d,ebx + ror r13d,5 + psrld xmm6,11 + xor r14d,edx + pxor xmm4,xmm5 + and r12d,r11d + xor r13d,r11d + pslld xmm5,11 + add ecx,DWORD PTR[52+rsp] + mov edi,edx + pxor xmm4,xmm6 + xor r12d,ebx + ror r14d,11 + movdqa xmm6,xmm7 + xor edi,r8d + add ecx,r12d + pxor xmm4,xmm5 + ror r13d,6 + and r15d,edi + xor r14d,edx + psrld xmm7,10 + add ecx,r13d + xor r15d,r8d + paddd xmm3,xmm4 + ror r14d,2 + add r10d,ecx + psrlq xmm6,17 + add ecx,r15d + mov r13d,r10d + add r14d,ecx + pxor xmm7,xmm6 + ror r13d,14 + mov ecx,r14d + mov r12d,r11d + ror r14d,9 + psrlq xmm6,2 + xor r13d,r10d + xor r12d,eax + pxor xmm7,xmm6 + ror r13d,5 + xor r14d,ecx + and r12d,r10d + pshufd xmm7,xmm7,128 + xor r13d,r10d + add ebx,DWORD PTR[56+rsp] + mov r15d,ecx + psrldq xmm7,8 + xor r12d,eax + ror r14d,11 + xor r15d,edx + add ebx,r12d + ror r13d,6 + paddd xmm3,xmm7 + and edi,r15d + xor r14d,ecx + add ebx,r13d + pshufd xmm7,xmm3,80 + xor edi,edx + ror r14d,2 + add r9d,ebx + movdqa xmm6,xmm7 + add ebx,edi + mov r13d,r9d + psrld xmm7,10 + add r14d,ebx + ror r13d,14 + psrlq xmm6,17 + mov ebx,r14d + mov r12d,r10d + pxor xmm7,xmm6 + ror r14d,9 + xor r13d,r9d + xor r12d,r11d + ror r13d,5 + xor r14d,ebx + psrlq xmm6,2 + and r12d,r9d + xor r13d,r9d + add eax,DWORD PTR[60+rsp] + pxor xmm7,xmm6 + mov edi,ebx + xor r12d,r11d + ror r14d,11 + pshufd xmm7,xmm7,8 + xor edi,ecx + add eax,r12d + movdqa xmm6,XMMWORD PTR[48+rsi] + ror r13d,6 + and r15d,edi + pslldq xmm7,8 + xor r14d,ebx + add eax,r13d + xor r15d,ecx + paddd xmm3,xmm7 + ror r14d,2 + add r8d,eax + add eax,r15d + paddd xmm6,xmm3 + mov r13d,r8d + add r14d,eax + movdqa XMMWORD PTR[48+rsp],xmm6 + cmp BYTE PTR[67+rsi],0 + jne $L$ssse3_00_47 + ror r13d,14 + mov eax,r14d + mov r12d,r9d + ror r14d,9 + xor r13d,r8d + xor r12d,r10d + ror r13d,5 + xor r14d,eax + and r12d,r8d + xor r13d,r8d + add r11d,DWORD PTR[rsp] + mov r15d,eax + xor r12d,r10d + ror r14d,11 + xor r15d,ebx + add r11d,r12d + ror r13d,6 + and edi,r15d + xor r14d,eax + add r11d,r13d + xor edi,ebx + ror r14d,2 + add edx,r11d + add r11d,edi + mov r13d,edx + add r14d,r11d + ror r13d,14 + mov r11d,r14d + mov r12d,r8d + ror r14d,9 + xor r13d,edx + xor r12d,r9d + ror r13d,5 + xor r14d,r11d + and r12d,edx + xor r13d,edx + add r10d,DWORD PTR[4+rsp] + mov edi,r11d + xor r12d,r9d + ror r14d,11 + xor edi,eax + add r10d,r12d + ror r13d,6 + and r15d,edi + xor r14d,r11d + add r10d,r13d + xor r15d,eax + ror r14d,2 + add ecx,r10d + add r10d,r15d + mov r13d,ecx + add r14d,r10d + ror r13d,14 + mov r10d,r14d + mov r12d,edx + ror r14d,9 + xor r13d,ecx + xor r12d,r8d + ror r13d,5 + xor r14d,r10d + and r12d,ecx + xor r13d,ecx + add r9d,DWORD PTR[8+rsp] + mov r15d,r10d + xor r12d,r8d + ror r14d,11 + xor r15d,r11d + add r9d,r12d + ror r13d,6 + and edi,r15d + xor r14d,r10d + add r9d,r13d + xor edi,r11d + ror r14d,2 + add ebx,r9d + add r9d,edi + mov r13d,ebx + add r14d,r9d + ror r13d,14 + mov r9d,r14d + mov r12d,ecx + ror r14d,9 + xor r13d,ebx + xor r12d,edx + ror r13d,5 + xor r14d,r9d + and r12d,ebx + xor r13d,ebx + add r8d,DWORD PTR[12+rsp] + mov edi,r9d + xor r12d,edx + ror r14d,11 + xor edi,r10d + add r8d,r12d + ror r13d,6 + and r15d,edi + xor r14d,r9d + add r8d,r13d + xor r15d,r10d + ror r14d,2 + add eax,r8d + add r8d,r15d + mov r13d,eax + add r14d,r8d + ror r13d,14 + mov r8d,r14d + mov r12d,ebx + ror r14d,9 + xor r13d,eax + xor r12d,ecx + ror r13d,5 + xor r14d,r8d + and r12d,eax + xor r13d,eax + add edx,DWORD PTR[16+rsp] + mov r15d,r8d + xor r12d,ecx + ror r14d,11 + xor r15d,r9d + add edx,r12d + ror r13d,6 + and edi,r15d + xor r14d,r8d + add edx,r13d + xor edi,r9d + ror r14d,2 + add r11d,edx + add edx,edi + mov r13d,r11d + add r14d,edx + ror r13d,14 + mov edx,r14d + mov r12d,eax + ror r14d,9 + xor r13d,r11d + xor r12d,ebx + ror r13d,5 + xor r14d,edx + and r12d,r11d + xor r13d,r11d + add ecx,DWORD PTR[20+rsp] + mov edi,edx + xor r12d,ebx + ror r14d,11 + xor edi,r8d + add ecx,r12d + ror r13d,6 + and r15d,edi + xor r14d,edx + add ecx,r13d + xor r15d,r8d + ror r14d,2 + add r10d,ecx + add ecx,r15d + mov r13d,r10d + add r14d,ecx + ror r13d,14 + mov ecx,r14d + mov r12d,r11d + ror r14d,9 + xor r13d,r10d + xor r12d,eax + ror r13d,5 + xor r14d,ecx + and r12d,r10d + xor r13d,r10d + add ebx,DWORD PTR[24+rsp] + mov r15d,ecx + xor r12d,eax + ror r14d,11 + xor r15d,edx + add ebx,r12d + ror r13d,6 + and edi,r15d + xor r14d,ecx + add ebx,r13d + xor edi,edx + ror r14d,2 + add r9d,ebx + add ebx,edi + mov r13d,r9d + add r14d,ebx + ror r13d,14 + mov ebx,r14d + mov r12d,r10d + ror r14d,9 + xor r13d,r9d + xor r12d,r11d + ror r13d,5 + xor r14d,ebx + and r12d,r9d + xor r13d,r9d + add eax,DWORD PTR[28+rsp] + mov edi,ebx + xor r12d,r11d + ror r14d,11 + xor edi,ecx + add eax,r12d + ror r13d,6 + and r15d,edi + xor r14d,ebx + add eax,r13d + xor r15d,ecx + ror r14d,2 + add r8d,eax + add eax,r15d + mov r13d,r8d + add r14d,eax + ror r13d,14 + mov eax,r14d + mov r12d,r9d + ror r14d,9 + xor r13d,r8d + xor r12d,r10d + ror r13d,5 + xor r14d,eax + and r12d,r8d + xor r13d,r8d + add r11d,DWORD PTR[32+rsp] + mov r15d,eax + xor r12d,r10d + ror r14d,11 + xor r15d,ebx + add r11d,r12d + ror r13d,6 + and edi,r15d + xor r14d,eax + add r11d,r13d + xor edi,ebx + ror r14d,2 + add edx,r11d + add r11d,edi + mov r13d,edx + add r14d,r11d + ror r13d,14 + mov r11d,r14d + mov r12d,r8d + ror r14d,9 + xor r13d,edx + xor r12d,r9d + ror r13d,5 + xor r14d,r11d + and r12d,edx + xor r13d,edx + add r10d,DWORD PTR[36+rsp] + mov edi,r11d + xor r12d,r9d + ror r14d,11 + xor edi,eax + add r10d,r12d + ror r13d,6 + and r15d,edi + xor r14d,r11d + add r10d,r13d + xor r15d,eax + ror r14d,2 + add ecx,r10d + add r10d,r15d + mov r13d,ecx + add r14d,r10d + ror r13d,14 + mov r10d,r14d + mov r12d,edx + ror r14d,9 + xor r13d,ecx + xor r12d,r8d + ror r13d,5 + xor r14d,r10d + and r12d,ecx + xor r13d,ecx + add r9d,DWORD PTR[40+rsp] + mov r15d,r10d + xor r12d,r8d + ror r14d,11 + xor r15d,r11d + add r9d,r12d + ror r13d,6 + and edi,r15d + xor r14d,r10d + add r9d,r13d + xor edi,r11d + ror r14d,2 + add ebx,r9d + add r9d,edi + mov r13d,ebx + add r14d,r9d + ror r13d,14 + mov r9d,r14d + mov r12d,ecx + ror r14d,9 + xor r13d,ebx + xor r12d,edx + ror r13d,5 + xor r14d,r9d + and r12d,ebx + xor r13d,ebx + add r8d,DWORD PTR[44+rsp] + mov edi,r9d + xor r12d,edx + ror r14d,11 + xor edi,r10d + add r8d,r12d + ror r13d,6 + and r15d,edi + xor r14d,r9d + add r8d,r13d + xor r15d,r10d + ror r14d,2 + add eax,r8d + add r8d,r15d + mov r13d,eax + add r14d,r8d + ror r13d,14 + mov r8d,r14d + mov r12d,ebx + ror r14d,9 + xor r13d,eax + xor r12d,ecx + ror r13d,5 + xor r14d,r8d + and r12d,eax + xor r13d,eax + add edx,DWORD PTR[48+rsp] + mov r15d,r8d + xor r12d,ecx + ror r14d,11 + xor r15d,r9d + add edx,r12d + ror r13d,6 + and edi,r15d + xor r14d,r8d + add edx,r13d + xor edi,r9d + ror r14d,2 + add r11d,edx + add edx,edi + mov r13d,r11d + add r14d,edx + ror r13d,14 + mov edx,r14d + mov r12d,eax + ror r14d,9 + xor r13d,r11d + xor r12d,ebx + ror r13d,5 + xor r14d,edx + and r12d,r11d + xor r13d,r11d + add ecx,DWORD PTR[52+rsp] + mov edi,edx + xor r12d,ebx + ror r14d,11 + xor edi,r8d + add ecx,r12d + ror r13d,6 + and r15d,edi + xor r14d,edx + add ecx,r13d + xor r15d,r8d + ror r14d,2 + add r10d,ecx + add ecx,r15d + mov r13d,r10d + add r14d,ecx + ror r13d,14 + mov ecx,r14d + mov r12d,r11d + ror r14d,9 + xor r13d,r10d + xor r12d,eax + ror r13d,5 + xor r14d,ecx + and r12d,r10d + xor r13d,r10d + add ebx,DWORD PTR[56+rsp] + mov r15d,ecx + xor r12d,eax + ror r14d,11 + xor r15d,edx + add ebx,r12d + ror r13d,6 + and edi,r15d + xor r14d,ecx + add ebx,r13d + xor edi,edx + ror r14d,2 + add r9d,ebx + add ebx,edi + mov r13d,r9d + add r14d,ebx + ror r13d,14 + mov ebx,r14d + mov r12d,r10d + ror r14d,9 + xor r13d,r9d + xor r12d,r11d + ror r13d,5 + xor r14d,ebx + and r12d,r9d + xor r13d,r9d + add eax,DWORD PTR[60+rsp] + mov edi,ebx + xor r12d,r11d + ror r14d,11 + xor edi,ecx + add eax,r12d + ror r13d,6 + and r15d,edi + xor r14d,ebx + add eax,r13d + xor r15d,ecx + ror r14d,2 + add r8d,eax + add eax,r15d + mov r13d,r8d + add r14d,eax + mov rdi,QWORD PTR[rbp] + mov eax,r14d + mov rsi,QWORD PTR[8+rbp] + + add eax,DWORD PTR[rdi] + add ebx,DWORD PTR[4+rdi] + add ecx,DWORD PTR[8+rdi] + add edx,DWORD PTR[12+rdi] + add r8d,DWORD PTR[16+rdi] + add r9d,DWORD PTR[20+rdi] + add r10d,DWORD PTR[24+rdi] + add r11d,DWORD PTR[28+rdi] + + lea rsi,QWORD PTR[64+rsi] + cmp rsi,QWORD PTR[16+rbp] + + mov DWORD PTR[rdi],eax + mov DWORD PTR[4+rdi],ebx + mov DWORD PTR[8+rdi],ecx + mov DWORD PTR[12+rdi],edx + mov DWORD PTR[16+rdi],r8d + mov DWORD PTR[20+rdi],r9d + mov DWORD PTR[24+rdi],r10d + mov DWORD PTR[28+rdi],r11d + jb $L$loop_ssse3 + + xorps xmm0,xmm0 + lea r11,QWORD PTR[((104+48))+rbp] + + movaps XMMWORD PTR[rsp],xmm0 + movaps XMMWORD PTR[16+rsp],xmm0 + movaps XMMWORD PTR[32+rsp],xmm0 + movaps XMMWORD PTR[48+rsp],xmm0 + movaps xmm6,XMMWORD PTR[32+rbp] + movaps xmm7,XMMWORD PTR[48+rbp] + movaps xmm8,XMMWORD PTR[64+rbp] + movaps xmm9,XMMWORD PTR[80+rbp] + mov r15,QWORD PTR[104+rbp] + + mov r14,QWORD PTR[((-40))+r11] + + mov r13,QWORD PTR[((-32))+r11] + + mov r12,QWORD PTR[((-24))+r11] + + mov rbx,QWORD PTR[((-16))+r11] + + mov rbp,QWORD PTR[((-8))+r11] + +$L$SEH_epilogue_blst_sha256_block_data_order:: + mov rdi,QWORD PTR[8+r11] ;WIN64 epilogue + mov rsi,QWORD PTR[16+r11] + + lea rsp,QWORD PTR[r11] + DB 0F3h,0C3h ;repret + +$L$SEH_end_blst_sha256_block_data_order:: +blst_sha256_block_data_order ENDP +PUBLIC blst_sha256_emit + + +ALIGN 16 +blst_sha256_emit PROC PUBLIC + DB 243,15,30,250 + mov r8,QWORD PTR[rdx] + mov r9,QWORD PTR[8+rdx] + mov r10,QWORD PTR[16+rdx] + bswap r8 + mov r11,QWORD PTR[24+rdx] + bswap r9 + mov DWORD PTR[4+rcx],r8d + bswap r10 + mov DWORD PTR[12+rcx],r9d + bswap r11 + mov DWORD PTR[20+rcx],r10d + shr r8,32 + mov DWORD PTR[28+rcx],r11d + shr r9,32 + mov DWORD PTR[rcx],r8d + shr r10,32 + mov DWORD PTR[8+rcx],r9d + shr r11,32 + mov DWORD PTR[16+rcx],r10d + mov DWORD PTR[24+rcx],r11d + DB 0F3h,0C3h ;repret +blst_sha256_emit ENDP + +PUBLIC blst_sha256_bcopy + + +ALIGN 16 +blst_sha256_bcopy PROC PUBLIC + DB 243,15,30,250 + sub rcx,rdx +$L$oop_bcopy:: + movzx eax,BYTE PTR[rdx] + lea rdx,QWORD PTR[1+rdx] + mov BYTE PTR[((-1))+rdx*1+rcx],al + dec r8 + jnz $L$oop_bcopy + DB 0F3h,0C3h ;repret +blst_sha256_bcopy ENDP + +PUBLIC blst_sha256_hcopy + + +ALIGN 16 +blst_sha256_hcopy PROC PUBLIC + DB 243,15,30,250 + mov r8,QWORD PTR[rdx] + mov r9,QWORD PTR[8+rdx] + mov r10,QWORD PTR[16+rdx] + mov r11,QWORD PTR[24+rdx] + mov QWORD PTR[rcx],r8 + mov QWORD PTR[8+rcx],r9 + mov QWORD PTR[16+rcx],r10 + mov QWORD PTR[24+rcx],r11 + DB 0F3h,0C3h ;repret +blst_sha256_hcopy ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_body_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_info_blst_sha256_block_data_order_shaext_prologue + + DD imagerel $L$SEH_body_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_epilogue_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_info_blst_sha256_block_data_order_shaext_body + + DD imagerel $L$SEH_epilogue_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_end_blst_sha256_block_data_order_shaext + DD imagerel $L$SEH_info_blst_sha256_block_data_order_shaext_epilogue + + DD imagerel $L$SEH_begin_blst_sha256_block_data_order + DD imagerel $L$SEH_body_blst_sha256_block_data_order + DD imagerel $L$SEH_info_blst_sha256_block_data_order_prologue + + DD imagerel $L$SEH_body_blst_sha256_block_data_order + DD imagerel $L$SEH_epilogue_blst_sha256_block_data_order + DD imagerel $L$SEH_info_blst_sha256_block_data_order_body + + DD imagerel $L$SEH_epilogue_blst_sha256_block_data_order + DD imagerel $L$SEH_end_blst_sha256_block_data_order + DD imagerel $L$SEH_info_blst_sha256_block_data_order_epilogue + +.pdata ENDS +.xdata SEGMENT READONLY ALIGN(8) +ALIGN 8 +$L$SEH_info_blst_sha256_block_data_order_shaext_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_blst_sha256_block_data_order_shaext_body:: +DB 1,0,15,0 +DB 000h,068h,000h,000h +DB 000h,078h,001h,000h +DB 000h,088h,002h,000h +DB 000h,098h,003h,000h +DB 000h,0a8h,004h,000h +DB 000h,074h,00ch,000h +DB 000h,064h,00dh,000h +DB 000h,0a2h +DB 000h,000h,000h,000h,000h,000h +$L$SEH_info_blst_sha256_block_data_order_shaext_epilogue:: +DB 1,0,5,11 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,003h +DB 000h,000h + +$L$SEH_info_blst_sha256_block_data_order_prologue:: +DB 1,0,5,00bh +DB 0,074h,1,0 +DB 0,064h,2,0 +DB 0,003h +DB 0,0 +$L$SEH_info_blst_sha256_block_data_order_body:: +DB 1,0,26,5 +DB 000h,068h,002h,000h +DB 000h,078h,003h,000h +DB 000h,088h,004h,000h +DB 000h,098h,005h,000h +DB 000h,0f4h,00dh,000h +DB 000h,0e4h,00eh,000h +DB 000h,0d4h,00fh,000h +DB 000h,0c4h,010h,000h +DB 000h,034h,011h,000h +DB 000h,074h,014h,000h +DB 000h,064h,015h,000h +DB 000h,003h +DB 000h,001h,012h,000h +DB 000h,050h +$L$SEH_info_blst_sha256_block_data_order_epilogue:: +DB 1,0,5,11 +DB 000h,074h,001h,000h +DB 000h,064h,002h,000h +DB 000h,003h +DB 000h,000h + + +.xdata ENDS +END diff --git a/crypto/blst_src/bulk_addition.c b/crypto/blst_src/bulk_addition.c new file mode 100644 index 00000000000..81afc530665 --- /dev/null +++ b/crypto/blst_src/bulk_addition.c @@ -0,0 +1,168 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" +#include "point.h" + +/* + * This implementation uses explicit addition formula: + * + * λ = (Y₂-Y₁)/(X₂-X₁) + * X₃ = λ²-(X₁+X₂) + * Y₃ = λ⋅(X₁-X₃)-Y₁ + * + * But since we don't know if we'll have to add point to itself, we need + * to eventually resort to corresponding doubling formula: + * + * λ = 3X₁²/2Y₁ + * X₃ = λ²-2X₁ + * Y₃ = λ⋅(X₁-X₃)-Y₁ + * + * The formulae use prohibitively expensive inversion, but whenever we + * have a lot of affine points to accumulate, we can amortize the cost + * by applying Montgomery's batch inversion approach. As a result, + * asymptotic[!] per-point cost for addition is as small as 5M+1S. For + * comparison, ptype##_dadd_affine takes 8M+5S. In practice, all things + * considered, the improvement coefficient varies from 60% to 85% + * depending on platform and curve. + * + * THIS IMPLEMENTATION IS *NOT* CONSTANT-TIME. [But if there is an + * application that requires constant time-ness, speak up!] + */ + +/* + * Calculate λ's numerator and denominator. + * + * input: A x1 y1 - + * B x2 y2 - + * output: + * if A!=B: A x1 y1 (x2-x1)*mul_acc + * B x2+x1 y2-y1 (x2-x1) + * + * if A==B: A x y 2y*mul_acc + * B 2x 3*x^2 2y + * + * if A==-B: A 0 0 1*mul_acc + * B 0 3*x^2 0 + */ +#define HEAD(ptype, bits, field, one) \ +static void ptype##_head(ptype AB[2], const vec##bits mul_acc) \ +{ \ + ptype *A = AB, *B = AB+1; \ + limb_t inf = vec_is_zero(A, sizeof(ptype##_affine)) | \ + vec_is_zero(B, sizeof(ptype##_affine)); \ + static const vec##bits zero = { 0 }; \ +\ + sub_##field(B->Z, B->X, A->X); /* X2-X1 */ \ + add_##field(B->X, B->X, A->X); /* X2+X1 */ \ + add_##field(A->Z, B->Y, A->Y); /* Y2+Y1 */ \ + sub_##field(B->Y, B->Y, A->Y); /* Y2-Y1 */ \ + if (vec_is_zero(B->Z, sizeof(B->Z))) { /* X2==X1 */ \ + inf = vec_is_zero(A->Z, sizeof(A->Z)); \ + vec_select(B->X, A->Z, B->X, sizeof(B->X), inf); \ + sqr_##field(B->Y, A->X); \ + mul_by_3_##field(B->Y, B->Y); /* 3*X1^2 */ \ + vec_copy(B->Z, A->Z, sizeof(B->Z)); /* 2*Y1 */ \ + } /* B->Y is numenator */ \ + /* B->Z is denominator */ \ + vec_select(A->X, B->X, A->X, sizeof(A->X), inf); \ + vec_select(A->Y, A->Z, A->Y, sizeof(A->Y), inf); \ + vec_select(A->Z, one, B->Z, sizeof(A->Z), inf); \ + vec_select(B->Z, zero, B->Z, sizeof(B->Z), inf); \ + if (mul_acc != NULL) \ + mul_##field(A->Z, A->Z, mul_acc); /* chain multiplication */\ +} + +/* + * Calculate λ and resulting coordinates. + * + * input: A x1 y1 - + * B x2+x1 nominator - + * lambda 1/denominator + * output: D x3=(nom/den)^2-(x2+x1) y3=(nom/den)(x1-x3)-y1 + */ +#define TAIL(ptype, bits, field, one) \ +static void ptype##_tail(ptype *D, ptype AB[2], vec##bits lambda) \ +{ \ + ptype *A = AB, *B = AB+1; \ + vec##bits llambda; \ + limb_t inf = vec_is_zero(B->Z, sizeof(B->Z)); \ +\ + mul_##field(lambda, lambda, B->Y); /* λ = (Y2-Y1)/(X2-X1) */ \ + /* alt. 3*X1^2/2*Y1 */ \ + sqr_##field(llambda, lambda); \ + sub_##field(D->X, llambda, B->X); /* X3 = λ^2-X1-X2 */ \ +\ + sub_##field(D->Y, A->X, D->X); \ + mul_##field(D->Y, D->Y, lambda); \ + sub_##field(D->Y, D->Y, A->Y); /* Y3 = λ*(X1-X3)-Y1 */ \ +\ + vec_select(D->X, A->X, D->X, 2*sizeof(D->X), inf); \ + vec_select(B->Z, one, B->Z, sizeof(B->Z), inf); \ +} + +/* + * |points[]| is volatile buffer with |X|s and |Y|s initially holding + * input affine coordinates, and with |Z|s being used as additional + * temporary storage [unrelated to Jacobian coordinates]. |sum| is + * in-/output, initialize to infinity accordingly. + */ +#define ADDITION_BTREE(prefix, ptype, bits, field, one) \ +HEAD(ptype, bits, field, one) \ +TAIL(ptype, bits, field, one) \ +static void ptype##s_accumulate(ptype *sum, ptype points[], size_t n) \ +{ \ + ptype *dst; \ + void *mul_acc; \ + size_t i; \ +\ + while (n >= 16) { \ + if (n & 1) \ + ptype##_dadd_affine(sum, sum, (const ptype##_affine *)points++); \ + n /= 2; \ + for (mul_acc = NULL, i = n; i--; mul_acc = points->Z, points += 2) \ + ptype##_head(points, mul_acc); \ +\ + reciprocal_##field(points[-2].Z, points[-2].Z); /* 1/∏ Zi */ \ +\ + for (dst = points, i = n; --i;) { \ + dst--; points -= 2; \ + mul_##field(points[-2].Z, points[0].Z, points[-2].Z); \ + ptype##_tail(dst, points, points[-2].Z); \ + mul_##field(points[-2].Z, points[0].Z, points[1].Z); \ + } \ + dst--; points -= 2; \ + ptype##_tail(dst, points, points[0].Z); \ + points = dst; \ + } \ + while (n--) \ + ptype##_dadd_affine(sum, sum, (const ptype##_affine *)points++); \ +} \ +\ +void prefix##s_add(ptype *sum, const ptype##_affine *const points[], \ + size_t npoints) \ +{ \ + /* Performance with 288K scratch is within 1-2-3% from optimal */ \ + const size_t stride = sizeof(ptype)==sizeof(POINTonE1) ? 2048 : 1024; \ + ptype *scratch = alloca((npoints > stride ? stride : npoints) * \ + sizeof(ptype)); \ + const ptype##_affine *point = NULL; \ +\ + vec_zero(sum, sizeof(*sum)); \ + while (npoints) { \ + size_t i, j = npoints > stride ? stride : npoints; \ + for (i=0; i<j; i++) { \ + point = *points ? *points++ : point+1; \ + vec_copy(&scratch[i], point, sizeof(*point)); \ + } \ + ptype##s_accumulate(sum, scratch, j); \ + npoints -= j; \ + } \ +} + +ADDITION_BTREE(blst_p1, POINTonE1, 384, fp, BLS12_381_Rx.p2) + +ADDITION_BTREE(blst_p2, POINTonE2, 384x, fp2, BLS12_381_Rx.p2) diff --git a/crypto/blst_src/bytes.h b/crypto/blst_src/bytes.h new file mode 100644 index 00000000000..af910ba8145 --- /dev/null +++ b/crypto/blst_src/bytes.h @@ -0,0 +1,152 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_BYTES_H__ +#define __BLS12_381_ASM_BYTES_H__ + +static inline void bytes_zero(unsigned char *a, size_t num) +{ + size_t i; + + for (i = 0; i < num; i++) + a[i] = 0; +} + +static inline void limbs_from_be_bytes(limb_t *restrict ret, + const unsigned char *in, size_t n) +{ + limb_t limb = 0; + + while(n--) { + limb <<= 8; + limb |= *in++; + /* + * 'if (n % sizeof(limb_t) == 0)' is omitted because it's cheaper + * to perform redundant stores than to pay penalty for + * mispredicted branch. Besides, some compilers unroll the + * loop and remove redundant stores to 'restict'-ed storage... + */ + ret[n / sizeof(limb_t)] = limb; + } +} + +static inline void be_bytes_from_limbs(unsigned char *out, const limb_t *in, + size_t n) +{ + limb_t limb; + + while(n--) { + limb = in[n / sizeof(limb_t)]; + *out++ = (unsigned char)(limb >> (8 * (n % sizeof(limb_t)))); + } +} + +static inline void limbs_from_le_bytes(limb_t *restrict ret, + const unsigned char *in, size_t n) +{ + limb_t limb = 0; + + while(n--) { + limb <<= 8; + limb |= in[n]; + /* + * 'if (n % sizeof(limb_t) == 0)' is omitted because it's cheaper + * to perform redundant stores than to pay penalty for + * mispredicted branch. Besides, some compilers unroll the + * loop and remove redundant stores to 'restict'-ed storage... + */ + ret[n / sizeof(limb_t)] = limb; + } +} + +static inline void le_bytes_from_limbs(unsigned char *out, const limb_t *in, + size_t n) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + limb_t limb; + size_t i, j, r; + + if ((uptr_t)out == (uptr_t)in && is_endian.little) + return; + + r = n % sizeof(limb_t); + n /= sizeof(limb_t); + + for(i = 0; i < n; i++) { + for (limb = in[i], j = 0; j < sizeof(limb_t); j++, limb >>= 8) + *out++ = (unsigned char)limb; + } + if (r) { + for (limb = in[i], j = 0; j < r; j++, limb >>= 8) + *out++ = (unsigned char)limb; + } +} + +static inline char hex_from_nibble(unsigned char nibble) +{ + int mask = (9 - (nibble &= 0xf)) >> 31; + return (char)(nibble + ((('a'-10) & mask) | ('0' & ~mask))); +} + +static unsigned char nibble_from_hex(char c) +{ + int mask, ret; + + mask = (('a'-c-1) & (c-1-'f')) >> 31; + ret = (10 + c - 'a') & mask; + mask = (('A'-c-1) & (c-1-'F')) >> 31; + ret |= (10 + c - 'A') & mask; + mask = (('0'-c-1) & (c-1-'9')) >> 31; + ret |= (c - '0') & mask; + mask = ((ret-1) & ~mask) >> 31; + ret |= 16 & mask; + + return (unsigned char)ret; +} + +static void bytes_from_hexascii(unsigned char *ret, size_t sz, const char *hex) +{ + size_t len; + unsigned char b = 0; + + if (hex[0]=='0' && (hex[1]=='x' || hex[1]=='X')) + hex += 2; + + for (len = 0; len<2*sz && nibble_from_hex(hex[len])<16; len++) ; + + bytes_zero(ret, sz); + + while(len--) { + b <<= 4; + b |= nibble_from_hex(*hex++); + if (len % 2 == 0) + ret[len / 2] = b; + } +} + +static void limbs_from_hexascii(limb_t *ret, size_t sz, const char *hex) +{ + size_t len; + limb_t limb = 0; + + if (hex[0]=='0' && (hex[1]=='x' || hex[1]=='X')) + hex += 2; + + for (len = 0; len<2*sz && nibble_from_hex(hex[len])<16; len++) ; + + vec_zero(ret, sz); + + while(len--) { + limb <<= 4; + limb |= nibble_from_hex(*hex++); + if (len % (2*sizeof(limb_t)) == 0) + ret[len / (2*sizeof(limb_t))] = limb; + } +} + +#endif diff --git a/crypto/blst_src/client_min_pk.c b/crypto/blst_src/client_min_pk.c new file mode 100644 index 00000000000..df11e3dae73 --- /dev/null +++ b/crypto/blst_src/client_min_pk.c @@ -0,0 +1,17 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +/*#include "keygen.c" +#include "e2.c" +#include "hash_to_field.c" +#include "map_to_g2.c" +#include "e1.c" +#include "exp.c" +#include "sqrt.c" +#include "recip.c" +#include "consts.c" +#include "vect.c" +#include "exports.c"*/ diff --git a/crypto/blst_src/client_min_sig.c b/crypto/blst_src/client_min_sig.c new file mode 100644 index 00000000000..fffbd5ad52d --- /dev/null +++ b/crypto/blst_src/client_min_sig.c @@ -0,0 +1,17 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +/*#include "keygen.c" +#include "e1.c" +#include "hash_to_field.c" +#include "map_to_g1.c" +#include "e2.c" +#include "exp.c" +#include "sqrt.c" +#include "recip.c" +#include "consts.c" +#include "vect.c" +#include "exports.c"*/ diff --git a/crypto/blst_src/consts.c b/crypto/blst_src/consts.c new file mode 100644 index 00000000000..021c878a258 --- /dev/null +++ b/crypto/blst_src/consts.c @@ -0,0 +1,36 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consts.h" + +/* z = -0xd201000000010000 */ +const vec384 BLS12_381_P = { /* (z-1)^2 * (z^4 - z^2 + 1)/3 + z */ + TO_LIMB_T(0xb9feffffffffaaab), TO_LIMB_T(0x1eabfffeb153ffff), + TO_LIMB_T(0x6730d2a0f6b0f624), TO_LIMB_T(0x64774b84f38512bf), + TO_LIMB_T(0x4b1ba7b6434bacd7), TO_LIMB_T(0x1a0111ea397fe69a) +}; +const limb_t BLS12_381_p0 = (limb_t)0x89f3fffcfffcfffd; /* -1/P */ + +const radix384 BLS12_381_Rx = { /* (1<<384)%P, "radix", one-in-Montgomery */ + { { ONE_MONT_P }, + { 0 } } +}; + +const vec384 BLS12_381_RR = { /* (1<<768)%P, "radix"^2, to-Montgomery */ + TO_LIMB_T(0xf4df1f341c341746), TO_LIMB_T(0x0a76e6a609d104f1), + TO_LIMB_T(0x8de5476c4c95b6d5), TO_LIMB_T(0x67eb88a9939d83c0), + TO_LIMB_T(0x9a793e85b519952d), TO_LIMB_T(0x11988fe592cae3aa) +}; + +const vec256 BLS12_381_r = { /* z^4 - z^2 + 1, group order */ + TO_LIMB_T(0xffffffff00000001), TO_LIMB_T(0x53bda402fffe5bfe), + TO_LIMB_T(0x3339d80809a1d805), TO_LIMB_T(0x73eda753299d7d48) +}; + +const vec256 BLS12_381_rRR = { /* (1<<512)%r, "radix"^2, to-Montgomery */ + TO_LIMB_T(0xc999e990f3f29c6d), TO_LIMB_T(0x2b6cedcb87925c23), + TO_LIMB_T(0x05d314967254398f), TO_LIMB_T(0x0748d9d99f59ff11) +}; diff --git a/crypto/blst_src/consts.h b/crypto/blst_src/consts.h new file mode 100644 index 00000000000..cb391b817df --- /dev/null +++ b/crypto/blst_src/consts.h @@ -0,0 +1,30 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_CONST_H__ +#define __BLS12_381_ASM_CONST_H__ +#include "vect.h" + +extern const vec384 BLS12_381_P; +extern const limb_t BLS12_381_p0; +static const limb_t p0 = (limb_t)0x89f3fffcfffcfffd; /* -1/P */ +typedef union { vec384 p12[12]; vec384x p2; vec384 p; } radix384; +extern const radix384 BLS12_381_Rx; /* (1<<384)%P, "radix", one-in-Montgomery */ +extern const vec384 BLS12_381_RR; /* (1<<768)%P, "radix"^2, to-Montgomery */ + +#define ONE_MONT_P TO_LIMB_T(0x760900000002fffd), \ + TO_LIMB_T(0xebf4000bc40c0002), \ + TO_LIMB_T(0x5f48985753c758ba), \ + TO_LIMB_T(0x77ce585370525745), \ + TO_LIMB_T(0x5c071a97a256ec6d), \ + TO_LIMB_T(0x15f65ec3fa80e493) + +#define ZERO_384 (BLS12_381_Rx.p2[1]) + +extern const vec256 BLS12_381_r; /* order */ +static const limb_t r0 = (limb_t)0xfffffffeffffffff; /* -1/r */ +extern const vec256 BLS12_381_rRR; /* (1<<512)%r, "radix"^2, to-Montgomery */ + +#endif diff --git a/crypto/blst_src/e1.c b/crypto/blst_src/e1.c new file mode 100644 index 00000000000..91c4cdbf39c --- /dev/null +++ b/crypto/blst_src/e1.c @@ -0,0 +1,564 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "point.h" +#include "fields.h" +#include "errors.h" + +/* + * y^2 = x^3 + B + */ +static const vec384 B_E1 = { /* (4 << 384) % P */ + TO_LIMB_T(0xaa270000000cfff3), TO_LIMB_T(0x53cc0032fc34000a), + TO_LIMB_T(0x478fe97a6b0a807f), TO_LIMB_T(0xb1d37ebee6ba24d7), + TO_LIMB_T(0x8ec9733bbf78ab2f), TO_LIMB_T(0x09d645513d83de7e) +}; + +const POINTonE1 BLS12_381_G1 = { /* generator point [in Montgomery] */ + /* (0x17f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905 + * a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb << 384) % P */ + { TO_LIMB_T(0x5cb38790fd530c16), TO_LIMB_T(0x7817fc679976fff5), + TO_LIMB_T(0x154f95c7143ba1c1), TO_LIMB_T(0xf0ae6acdf3d0e747), + TO_LIMB_T(0xedce6ecc21dbf440), TO_LIMB_T(0x120177419e0bfb75) }, + /* (0x08b3f481e3aaa0f1a09e30ed741d8ae4fcf5e095d5d00af6 + * 00db18cb2c04b3edd03cc744a2888ae40caa232946c5e7e1 << 384) % P */ + { TO_LIMB_T(0xbaac93d50ce72271), TO_LIMB_T(0x8c22631a7918fd8e), + TO_LIMB_T(0xdd595f13570725ce), TO_LIMB_T(0x51ac582950405194), + TO_LIMB_T(0x0e1c8c3fad0059c0), TO_LIMB_T(0x0bbc3efc5008a26a) }, + { ONE_MONT_P } +}; + +const POINTonE1 BLS12_381_NEG_G1 = { /* negative generator [in Montgomery] */ + /* (0x17f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905 + * a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb << 384) % P */ + { TO_LIMB_T(0x5cb38790fd530c16), TO_LIMB_T(0x7817fc679976fff5), + TO_LIMB_T(0x154f95c7143ba1c1), TO_LIMB_T(0xf0ae6acdf3d0e747), + TO_LIMB_T(0xedce6ecc21dbf440), TO_LIMB_T(0x120177419e0bfb75) }, + /* (0x114d1d6855d545a8aa7d76c8cf2e21f267816aef1db507c9 + * 6655b9d5caac42364e6f38ba0ecb751bad54dcd6b939c2ca << 384) % P */ + { TO_LIMB_T(0xff526c2af318883a), TO_LIMB_T(0x92899ce4383b0270), + TO_LIMB_T(0x89d7738d9fa9d055), TO_LIMB_T(0x12caf35ba344c12a), + TO_LIMB_T(0x3cff1b76964b5317), TO_LIMB_T(0x0e44d2ede9774430) }, + { ONE_MONT_P } +}; + +static inline void mul_by_b_onE1(vec384 out, const vec384 in) +{ lshift_fp(out, in, 2); } + +static inline void mul_by_4b_onE1(vec384 out, const vec384 in) +{ lshift_fp(out, in, 4); } + +static void POINTonE1_cneg(POINTonE1 *p, bool_t cbit) +{ cneg_fp(p->Y, p->Y, cbit); } + +void blst_p1_cneg(POINTonE1 *a, int cbit) +{ POINTonE1_cneg(a, is_zero(cbit) ^ 1); } + +static void POINTonE1_from_Jacobian(POINTonE1 *out, const POINTonE1 *in) +{ + vec384 Z, ZZ; + limb_t inf = vec_is_zero(in->Z, sizeof(in->Z)); + + reciprocal_fp(Z, in->Z); /* 1/Z */ + + sqr_fp(ZZ, Z); + mul_fp(out->X, in->X, ZZ); /* X = X/Z^2 */ + + mul_fp(ZZ, ZZ, Z); + mul_fp(out->Y, in->Y, ZZ); /* Y = Y/Z^3 */ + + vec_select(out->Z, in->Z, BLS12_381_G1.Z, + sizeof(BLS12_381_G1.Z), inf); /* Z = inf ? 0 : 1 */ +} + +void blst_p1_from_jacobian(POINTonE1 *out, const POINTonE1 *a) +{ POINTonE1_from_Jacobian(out, a); } + +static void POINTonE1_to_affine(POINTonE1_affine *out, const POINTonE1 *in) +{ + POINTonE1 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p, sizeof(in->Z))) { + POINTonE1_from_Jacobian(&p, in); + in = &p; + } + vec_copy(out, in, sizeof(*out)); +} + +void blst_p1_to_affine(POINTonE1_affine *out, const POINTonE1 *a) +{ POINTonE1_to_affine(out, a); } + +void blst_p1_from_affine(POINTonE1 *out, const POINTonE1_affine *a) +{ + vec_copy(out, a, sizeof(*a)); + vec_select(out->Z, a->X, BLS12_381_Rx.p, sizeof(out->Z), + vec_is_zero(a, sizeof(*a))); +} + +static bool_t POINTonE1_affine_on_curve(const POINTonE1_affine *p) +{ + vec384 XXX, YY; + + sqr_fp(XXX, p->X); + mul_fp(XXX, XXX, p->X); /* X^3 */ + add_fp(XXX, XXX, B_E1); /* X^3 + B */ + + sqr_fp(YY, p->Y); /* Y^2 */ + + return vec_is_equal(XXX, YY, sizeof(XXX)); +} + +int blst_p1_affine_on_curve(const POINTonE1_affine *p) +{ return (int)(POINTonE1_affine_on_curve(p) | vec_is_zero(p, sizeof(*p))); } + +static bool_t POINTonE1_on_curve(const POINTonE1 *p) +{ + vec384 XXX, YY, BZ6; + limb_t inf = vec_is_zero(p->Z, sizeof(p->Z)); + + sqr_fp(BZ6, p->Z); + mul_fp(BZ6, BZ6, p->Z); + sqr_fp(BZ6, BZ6); /* Z^6 */ + mul_by_b_onE1(BZ6, BZ6); /* B*Z^6 */ + + sqr_fp(XXX, p->X); + mul_fp(XXX, XXX, p->X); /* X^3 */ + add_fp(XXX, XXX, BZ6); /* X^3 + B*Z^6 */ + + sqr_fp(YY, p->Y); /* Y^2 */ + + return vec_is_equal(XXX, YY, sizeof(XXX)) | inf; +} + +int blst_p1_on_curve(const POINTonE1 *p) +{ return (int)POINTonE1_on_curve(p); } + +static limb_t POINTonE1_affine_Serialize_BE(unsigned char out[96], + const POINTonE1_affine *in) +{ + vec384 temp; + + from_fp(temp, in->X); + be_bytes_from_limbs(out, temp, sizeof(temp)); + + from_fp(temp, in->Y); + be_bytes_from_limbs(out + 48, temp, sizeof(temp)); + + return sgn0_pty_mod_384(temp, BLS12_381_P); +} + +void blst_p1_affine_serialize(unsigned char out[96], + const POINTonE1_affine *in) +{ + if (vec_is_zero(in->X, 2*sizeof(in->X))) { + bytes_zero(out, 96); + out[0] = 0x40; /* infinitiy bit */ + } else { + (void)POINTonE1_affine_Serialize_BE(out, in); + } +} + +static limb_t POINTonE1_Serialize_BE(unsigned char out[96], + const POINTonE1 *in) +{ + POINTonE1 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p, sizeof(in->Z))) { + POINTonE1_from_Jacobian(&p, in); + in = &p; + } + + return POINTonE1_affine_Serialize_BE(out, (const POINTonE1_affine *)in); +} + +static void POINTonE1_Serialize(unsigned char out[96], const POINTonE1 *in) +{ + if (vec_is_zero(in->Z, sizeof(in->Z))) { + bytes_zero(out, 96); + out[0] = 0x40; /* infinitiy bit */ + } else { + (void)POINTonE1_Serialize_BE(out, in); + } +} + +void blst_p1_serialize(unsigned char out[96], const POINTonE1 *in) +{ POINTonE1_Serialize(out, in); } + +static limb_t POINTonE1_affine_Compress_BE(unsigned char out[48], + const POINTonE1_affine *in) +{ + vec384 temp; + + from_fp(temp, in->X); + be_bytes_from_limbs(out, temp, sizeof(temp)); + + return sgn0_pty_mont_384(in->Y, BLS12_381_P, p0); +} + +void blst_p1_affine_compress(unsigned char out[48], const POINTonE1_affine *in) +{ + if (vec_is_zero(in->X, 2*sizeof(in->X))) { + bytes_zero(out, 48); + out[0] = 0xc0; /* compressed and infinitiy bits */ + } else { + limb_t sign = POINTonE1_affine_Compress_BE(out, in); + out[0] |= (unsigned char)(0x80 | ((sign & 2) << 4)); + } +} + +static limb_t POINTonE1_Compress_BE(unsigned char out[48], + const POINTonE1 *in) +{ + POINTonE1 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p, sizeof(in->Z))) { + POINTonE1_from_Jacobian(&p, in); + in = &p; + } + + return POINTonE1_affine_Compress_BE(out, (const POINTonE1_affine *)in); +} + +void blst_p1_compress(unsigned char out[48], const POINTonE1 *in) +{ + if (vec_is_zero(in->Z, sizeof(in->Z))) { + bytes_zero(out, 48); + out[0] = 0xc0; /* compressed and infinitiy bits */ + } else { + limb_t sign = POINTonE1_Compress_BE(out, in); + out[0] |= (unsigned char)(0x80 | ((sign & 2) << 4)); + } +} + +static limb_t POINTonE1_Uncompress_BE(POINTonE1_affine *out, + const unsigned char in[48]) +{ + POINTonE1_affine ret; + vec384 temp; + + limbs_from_be_bytes(ret.X, in, sizeof(ret.X)); + /* clear top 3 bits in case caller was conveying some information there */ + ret.X[sizeof(ret.X)/sizeof(limb_t)-1] &= ((limb_t)0-1) >> 3; + add_fp(temp, ret.X, ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X, sizeof(temp))) + return (limb_t)0 - BLST_BAD_ENCODING; + mul_fp(ret.X, ret.X, BLS12_381_RR); + + sqr_fp(ret.Y, ret.X); + mul_fp(ret.Y, ret.Y, ret.X); + add_fp(ret.Y, ret.Y, B_E1); /* X^3 + B */ + if (!sqrt_fp(ret.Y, ret.Y)) + return (limb_t)0 - BLST_POINT_NOT_ON_CURVE; + + vec_copy(out, &ret, sizeof(ret)); + + return sgn0_pty_mont_384(out->Y, BLS12_381_P, p0); +} + +static BLST_ERROR POINTonE1_Uncompress_Z(POINTonE1_affine *out, + const unsigned char in[48]) +{ + unsigned char in0 = in[0]; + limb_t sgn0_pty; + + if ((in0 & 0x80) == 0) /* compressed bit */ + return BLST_BAD_ENCODING; + + if (in0 & 0x40) { /* infinity bit */ + if (byte_is_zero(in0 & 0x3f) & bytes_are_zero(in+1, 47)) { + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } else { + return BLST_BAD_ENCODING; + } + } + + sgn0_pty = POINTonE1_Uncompress_BE(out, in); + + if (sgn0_pty > 3) + return (BLST_ERROR)(0 - sgn0_pty); /* POINT_NOT_ON_CURVE */ + + sgn0_pty >>= 1; /* skip over parity bit */ + sgn0_pty ^= (in0 & 0x20) >> 5; + cneg_fp(out->Y, out->Y, sgn0_pty); + + /* (0,±2) is not in group, but application might want to ignore? */ + return vec_is_zero(out->X, sizeof(out->X)) ? BLST_POINT_NOT_IN_GROUP + : BLST_SUCCESS; +} + +BLST_ERROR blst_p1_uncompress(POINTonE1_affine *out, const unsigned char in[48]) +{ return POINTonE1_Uncompress_Z(out, in); } + +static BLST_ERROR POINTonE1_Deserialize_BE(POINTonE1_affine *out, + const unsigned char in[96]) +{ + POINTonE1_affine ret; + vec384 temp; + + limbs_from_be_bytes(ret.X, in, sizeof(ret.X)); + limbs_from_be_bytes(ret.Y, in + 48, sizeof(ret.Y)); + + /* clear top 3 bits in case caller was conveying some information there */ + ret.X[sizeof(ret.X)/sizeof(limb_t)-1] &= ((limb_t)0-1) >> 3; + add_fp(temp, ret.X, ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X, sizeof(temp))) + return BLST_BAD_ENCODING; + + add_fp(temp, ret.Y, ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.Y, sizeof(temp))) + return BLST_BAD_ENCODING; + + mul_fp(ret.X, ret.X, BLS12_381_RR); + mul_fp(ret.Y, ret.Y, BLS12_381_RR); + + if (!POINTonE1_affine_on_curve(&ret)) + return BLST_POINT_NOT_ON_CURVE; + + vec_copy(out, &ret, sizeof(ret)); + + /* (0,±2) is not in group, but application might want to ignore? */ + return vec_is_zero(out->X, sizeof(out->X)) ? BLST_POINT_NOT_IN_GROUP + : BLST_SUCCESS; +} + +static BLST_ERROR POINTonE1_Deserialize_Z(POINTonE1_affine *out, + const unsigned char in[96]) +{ + unsigned char in0 = in[0]; + + if ((in0 & 0xe0) == 0) + return POINTonE1_Deserialize_BE(out, in); + + if (in0 & 0x80) /* compressed bit */ + return POINTonE1_Uncompress_Z(out, in); + + if (in0 & 0x40) { /* infinity bit */ + if (byte_is_zero(in0 & 0x3f) & bytes_are_zero(in+1, 95)) { + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } + } + + return BLST_BAD_ENCODING; +} + +BLST_ERROR blst_p1_deserialize(POINTonE1_affine *out, + const unsigned char in[96]) +{ return POINTonE1_Deserialize_Z(out, in); } + +#include "ec_ops.h" +POINT_DADD_IMPL(POINTonE1, 384, fp) +POINT_DADD_AFFINE_IMPL_A0(POINTonE1, 384, fp, BLS12_381_Rx.p) +POINT_ADD_IMPL(POINTonE1, 384, fp) +POINT_ADD_AFFINE_IMPL(POINTonE1, 384, fp, BLS12_381_Rx.p) +POINT_DOUBLE_IMPL_A0(POINTonE1, 384, fp) +POINT_IS_EQUAL_IMPL(POINTonE1, 384, fp) + +void blst_p1_add(POINTonE1 *out, const POINTonE1 *a, const POINTonE1 *b) +{ POINTonE1_add(out, a, b); } + +void blst_p1_add_or_double(POINTonE1 *out, const POINTonE1 *a, + const POINTonE1 *b) +{ POINTonE1_dadd(out, a, b, NULL); } + +void blst_p1_add_affine(POINTonE1 *out, const POINTonE1 *a, + const POINTonE1_affine *b) +{ POINTonE1_add_affine(out, a, b); } + +void blst_p1_add_or_double_affine(POINTonE1 *out, const POINTonE1 *a, + const POINTonE1_affine *b) +{ POINTonE1_dadd_affine(out, a, b); } + +void blst_p1_double(POINTonE1 *out, const POINTonE1 *a) +{ POINTonE1_double(out, a); } + +int blst_p1_is_equal(const POINTonE1 *a, const POINTonE1 *b) +{ return (int)POINTonE1_is_equal(a, b); } + +#include "ec_mult.h" +POINT_MULT_SCALAR_WX_IMPL(POINTonE1, 4) +POINT_MULT_SCALAR_WX_IMPL(POINTonE1, 5) + +#ifdef __BLST_PRIVATE_TESTMODE__ +POINT_AFFINE_MULT_SCALAR_IMPL(POINTonE1) + +DECLARE_PRIVATE_POINTXZ(POINTonE1, 384) +POINT_LADDER_PRE_IMPL(POINTonE1, 384, fp) +POINT_LADDER_STEP_IMPL_A0(POINTonE1, 384, fp, onE1) +POINT_LADDER_POST_IMPL_A0(POINTonE1, 384, fp, onE1) +POINT_MULT_SCALAR_LADDER_IMPL(POINTonE1) +#endif + +static const vec384 beta = { /* such that beta^3 - 1 = 0 */ + /* -1/2 * (1 + sqrt(-3)) = ((P-2)^(P-2)) * (1 + (P-3)^((P+1)/4)) */ + /* (0x1a0111ea397fe699ec02408663d4de85aa0d857d89759ad4 + 897d29650fb85f9b409427eb4f49fffd8bfd00000000aaac << 384) % P */ + TO_LIMB_T(0xcd03c9e48671f071), TO_LIMB_T(0x5dab22461fcda5d2), + TO_LIMB_T(0x587042afd3851b95), TO_LIMB_T(0x8eb60ebe01bacb9e), + TO_LIMB_T(0x03f97d6e83d050d2), TO_LIMB_T(0x18f0206554638741) +}; + +static void sigma(POINTonE1 *out, const POINTonE1 *in) +{ + vec_copy(out->X, in->X, 2*sizeof(out->X)); + mul_fp(out->Z, in->Z, beta); +} + +/* Gallant-Lambert-Vanstone, ~45% faster than POINTonE1_mult_w5 */ +static void POINTonE1_mult_glv(POINTonE1 *out, const POINTonE1 *in, + const pow256 SK) +{ + union { vec256 l; pow256 s; } val; + + /* SK/z^2 [in constant time] */ + + limbs_from_le_bytes(val.l, SK, 32); + div_by_zz(val.l); + le_bytes_from_limbs(val.s, val.l, 32); + + { + const byte *scalars[2] = { val.s+16, val.s }; + POINTonE1 table[2][1<<(5-1)]; /* 4.5KB */ + size_t i; + + POINTonE1_precompute_w5(table[0], in); + for (i = 0; i < 1<<(5-1); i++) { + mul_fp(table[1][i].X, table[0][i].X, beta); + cneg_fp(table[1][i].Y, table[0][i].Y, 1); + vec_copy(table[1][i].Z, table[0][i].Z, sizeof(table[1][i].Z)); + } + + POINTonE1s_mult_w5(out, NULL, 2, scalars, 128, table); + POINTonE1_cneg(out, 1); + mul_fp(out->Z, out->Z, beta); + mul_fp(out->Z, out->Z, beta); + } + + vec_zero(val.l, sizeof(val)); /* scrub the copy of SK */ +} + +static void POINTonE1_sign(POINTonE1 *out, const POINTonE1 *in, const pow256 SK) +{ + vec384 Z, ZZ; + limb_t inf; + + POINTonE1_mult_glv(out, in, SK); + + /* convert to affine to remove possible bias in out->Z */ + inf = vec_is_zero(out->Z, sizeof(out->Z)); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + flt_reciprocal_fp(Z, out->Z); /* 1/Z */ +#else + reciprocal_fp(Z, out->Z); /* 1/Z */ +#endif + + sqr_fp(ZZ, Z); + mul_fp(out->X, out->X, ZZ); /* X = X/Z^2 */ + + mul_fp(ZZ, ZZ, Z); + mul_fp(out->Y, out->Y, ZZ); /* Y = Y/Z^3 */ + + vec_select(out->Z, out->Z, BLS12_381_G1.Z, sizeof(BLS12_381_G1.Z), + inf); /* Z = inf ? 0 : 1 */ +} + +void blst_sk_to_pk_in_g1(POINTonE1 *out, const pow256 SK) +{ POINTonE1_sign(out, &BLS12_381_G1, SK); } + +void blst_sign_pk_in_g2(POINTonE1 *out, const POINTonE1 *msg, const pow256 SK) +{ POINTonE1_sign(out, msg, SK); } + +void blst_sk_to_pk2_in_g1(unsigned char out[96], POINTonE1_affine *PK, + const pow256 SK) +{ + POINTonE1 P[1]; + + POINTonE1_sign(P, &BLS12_381_G1, SK); + if (PK != NULL) + vec_copy(PK, P, sizeof(*PK)); + if (out != NULL) { + limb_t sgn0_pty = POINTonE1_Serialize_BE(out, P); + out[0] |= (sgn0_pty & 2) << 4; /* pre-decorate */ + out[0] |= vec_is_zero(P->Z, sizeof(P->Z)) << 6; + } +} + +void blst_sign_pk2_in_g2(unsigned char out[96], POINTonE1_affine *sig, + const POINTonE1 *hash, const pow256 SK) +{ + POINTonE1 P[1]; + + POINTonE1_sign(P, hash, SK); + if (sig != NULL) + vec_copy(sig, P, sizeof(*sig)); + if (out != NULL) { + limb_t sgn0_pty = POINTonE1_Serialize_BE(out, P); + out[0] |= (sgn0_pty & 2) << 4; /* pre-decorate */ + out[0] |= vec_is_zero(P->Z, sizeof(P->Z)) << 6; + } +} + +void blst_p1_mult(POINTonE1 *out, const POINTonE1 *a, + const byte *scalar, size_t nbits) +{ + if (nbits < 176) { + if (nbits) + POINTonE1_mult_w4(out, a, scalar, nbits); + else + vec_zero(out, sizeof(*out)); + } else if (nbits <= 256) { + union { vec256 l; pow256 s; } val; + size_t i, j, top, mask = (size_t)0 - 1; + + /* this is not about constant-time-ness, but branch optimization */ + for (top = (nbits + 7)/8, i=0, j=0; i<sizeof(val.s);) { + val.s[i++] = scalar[j] & mask; + mask = 0 - ((i - top) >> (8*sizeof(top)-1)); + j += 1 & mask; + } + + if (check_mod_256(val.s, BLS12_381_r)) /* z^4 is the formal limit */ + POINTonE1_mult_glv(out, a, val.s); + else /* should never be the case, added for formal completeness */ + POINTonE1_mult_w5(out, a, scalar, nbits); + + vec_zero(val.l, sizeof(val)); + } else { /* should never be the case, added for formal completeness */ + POINTonE1_mult_w5(out, a, scalar, nbits); + } +} + +void blst_p1_unchecked_mult(POINTonE1 *out, const POINTonE1 *a, + const byte *scalar, size_t nbits) +{ + if (nbits) + POINTonE1_mult_w4(out, a, scalar, nbits); + else + vec_zero(out, sizeof(*out)); +} + +int blst_p1_affine_is_equal(const POINTonE1_affine *a, + const POINTonE1_affine *b) +{ return (int)vec_is_equal(a, b, sizeof(*a)); } + +int blst_p1_is_inf(const POINTonE1 *p) +{ return (int)vec_is_zero(p->Z, sizeof(p->Z)); } + +const POINTonE1 *blst_p1_generator(void) +{ return &BLS12_381_G1; } + +int blst_p1_affine_is_inf(const POINTonE1_affine *p) +{ return (int)vec_is_zero(p, sizeof(*p)); } + +const POINTonE1_affine *blst_p1_affine_generator(void) +{ return (const POINTonE1_affine *)&BLS12_381_G1; } + +size_t blst_p1_sizeof(void) +{ return sizeof(POINTonE1); } + +size_t blst_p1_affine_sizeof(void) +{ return sizeof(POINTonE1_affine); } diff --git a/crypto/blst_src/e2.c b/crypto/blst_src/e2.c new file mode 100644 index 00000000000..822ac23c694 --- /dev/null +++ b/crypto/blst_src/e2.c @@ -0,0 +1,638 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "point.h" +#include "fields.h" +#include "errors.h" + +/* + * y^2 = x^3 + B + */ +static const vec384x B_E2 = { /* 4 + 4*i */ + { TO_LIMB_T(0xaa270000000cfff3), TO_LIMB_T(0x53cc0032fc34000a), + TO_LIMB_T(0x478fe97a6b0a807f), TO_LIMB_T(0xb1d37ebee6ba24d7), + TO_LIMB_T(0x8ec9733bbf78ab2f), TO_LIMB_T(0x09d645513d83de7e) }, + { TO_LIMB_T(0xaa270000000cfff3), TO_LIMB_T(0x53cc0032fc34000a), + TO_LIMB_T(0x478fe97a6b0a807f), TO_LIMB_T(0xb1d37ebee6ba24d7), + TO_LIMB_T(0x8ec9733bbf78ab2f), TO_LIMB_T(0x09d645513d83de7e) } +}; + +const POINTonE2 BLS12_381_G2 = { /* generator point [in Montgomery] */ +{ /* (0x024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02 + b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8 << 384) % P */ + { TO_LIMB_T(0xf5f28fa202940a10), TO_LIMB_T(0xb3f5fb2687b4961a), + TO_LIMB_T(0xa1a893b53e2ae580), TO_LIMB_T(0x9894999d1a3caee9), + TO_LIMB_T(0x6f67b7631863366b), TO_LIMB_T(0x058191924350bcd7) }, + /* (0x13e02b6052719f607dacd3a088274f65596bd0d09920b61a + b5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e << 384) % P */ + { TO_LIMB_T(0xa5a9c0759e23f606), TO_LIMB_T(0xaaa0c59dbccd60c3), + TO_LIMB_T(0x3bb17e18e2867806), TO_LIMB_T(0x1b1ab6cc8541b367), + TO_LIMB_T(0xc2b6ed0ef2158547), TO_LIMB_T(0x11922a097360edf3) } +}, +{ /* (0x0ce5d527727d6e118cc9cdc6da2e351aadfd9baa8cbdd3a7 + 6d429a695160d12c923ac9cc3baca289e193548608b82801 << 384) % P */ + { TO_LIMB_T(0x4c730af860494c4a), TO_LIMB_T(0x597cfa1f5e369c5a), + TO_LIMB_T(0xe7e6856caa0a635a), TO_LIMB_T(0xbbefb5e96e0d495f), + TO_LIMB_T(0x07d3a975f0ef25a2), TO_LIMB_T(0x0083fd8e7e80dae5) }, + /* (0x0606c4a02ea734cc32acd2b02bc28b99cb3e287e85a763af + 267492ab572e99ab3f370d275cec1da1aaa9075ff05f79be << 384) % P */ + { TO_LIMB_T(0xadc0fc92df64b05d), TO_LIMB_T(0x18aa270a2b1461dc), + TO_LIMB_T(0x86adac6a3be4eba0), TO_LIMB_T(0x79495c4ec93da33a), + TO_LIMB_T(0xe7175850a43ccaed), TO_LIMB_T(0x0b2bc2a163de1bf2) }, +}, +{ { ONE_MONT_P }, { 0 } } +}; + +const POINTonE2 BLS12_381_NEG_G2 = { /* negative generator [in Montgomery] */ +{ /* (0x024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02 + b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8 << 384) % P */ + { TO_LIMB_T(0xf5f28fa202940a10), TO_LIMB_T(0xb3f5fb2687b4961a), + TO_LIMB_T(0xa1a893b53e2ae580), TO_LIMB_T(0x9894999d1a3caee9), + TO_LIMB_T(0x6f67b7631863366b), TO_LIMB_T(0x058191924350bcd7) }, + /* (0x13e02b6052719f607dacd3a088274f65596bd0d09920b61a + b5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e << 384) % P */ + { TO_LIMB_T(0xa5a9c0759e23f606), TO_LIMB_T(0xaaa0c59dbccd60c3), + TO_LIMB_T(0x3bb17e18e2867806), TO_LIMB_T(0x1b1ab6cc8541b367), + TO_LIMB_T(0xc2b6ed0ef2158547), TO_LIMB_T(0x11922a097360edf3) } +}, +{ /* (0x0d1b3cc2c7027888be51d9ef691d77bcb679afda66c73f17 + f9ee3837a55024f78c71363275a75d75d86bab79f74782aa << 384) % P */ + { TO_LIMB_T(0x6d8bf5079fb65e61), TO_LIMB_T(0xc52f05df531d63a5), + TO_LIMB_T(0x7f4a4d344ca692c9), TO_LIMB_T(0xa887959b8577c95f), + TO_LIMB_T(0x4347fe40525c8734), TO_LIMB_T(0x197d145bbaff0bb5) }, + /* (0x13fa4d4a0ad8b1ce186ed5061789213d993923066dddaf10 + 40bc3ff59f825c78df74f2d75467e25e0f55f8a00fa030ed << 384) % P */ + { TO_LIMB_T(0x0c3e036d209afa4e), TO_LIMB_T(0x0601d8f4863f9e23), + TO_LIMB_T(0xe0832636bacc0a84), TO_LIMB_T(0xeb2def362a476f84), + TO_LIMB_T(0x64044f659f0ee1e9), TO_LIMB_T(0x0ed54f48d5a1caa7) } +}, +{ { ONE_MONT_P }, { 0 } } +}; + +static void mul_by_b_onE2(vec384x out, const vec384x in) +{ + sub_fp(out[0], in[0], in[1]); + add_fp(out[1], in[0], in[1]); + lshift_fp(out[0], out[0], 2); + lshift_fp(out[1], out[1], 2); +} + +static void mul_by_4b_onE2(vec384x out, const vec384x in) +{ + sub_fp(out[0], in[0], in[1]); + add_fp(out[1], in[0], in[1]); + lshift_fp(out[0], out[0], 4); + lshift_fp(out[1], out[1], 4); +} + +static void POINTonE2_cneg(POINTonE2 *p, bool_t cbit) +{ cneg_fp2(p->Y, p->Y, cbit); } + +void blst_p2_cneg(POINTonE2 *a, int cbit) +{ POINTonE2_cneg(a, is_zero(cbit) ^ 1); } + +static void POINTonE2_from_Jacobian(POINTonE2 *out, const POINTonE2 *in) +{ + vec384x Z, ZZ; + limb_t inf = vec_is_zero(in->Z, sizeof(in->Z)); + + reciprocal_fp2(Z, in->Z); /* 1/Z */ + + sqr_fp2(ZZ, Z); + mul_fp2(out->X, in->X, ZZ); /* X = X/Z^2 */ + + mul_fp2(ZZ, ZZ, Z); + mul_fp2(out->Y, in->Y, ZZ); /* Y = Y/Z^3 */ + + vec_select(out->Z, in->Z, BLS12_381_G2.Z, + sizeof(BLS12_381_G2.Z), inf); /* Z = inf ? 0 : 1 */ +} + +void blst_p2_from_jacobian(POINTonE2 *out, const POINTonE2 *a) +{ POINTonE2_from_Jacobian(out, a); } + +static void POINTonE2_to_affine(POINTonE2_affine *out, const POINTonE2 *in) +{ + POINTonE2 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p2, sizeof(in->Z))) { + POINTonE2_from_Jacobian(&p, in); + in = &p; + } + vec_copy(out, in, sizeof(*out)); +} + +void blst_p2_to_affine(POINTonE2_affine *out, const POINTonE2 *a) +{ POINTonE2_to_affine(out, a); } + +void blst_p2_from_affine(POINTonE2 *out, const POINTonE2_affine *a) +{ + vec_copy(out, a, sizeof(*a)); + vec_select(out->Z, a->X, BLS12_381_Rx.p2, sizeof(out->Z), + vec_is_zero(a, sizeof(*a))); +} + +static bool_t POINTonE2_affine_on_curve(const POINTonE2_affine *p) +{ + vec384x XXX, YY; + + sqr_fp2(XXX, p->X); + mul_fp2(XXX, XXX, p->X); /* X^3 */ + add_fp2(XXX, XXX, B_E2); /* X^3 + B */ + + sqr_fp2(YY, p->Y); /* Y^2 */ + + return vec_is_equal(XXX, YY, sizeof(XXX)); +} + +int blst_p2_affine_on_curve(const POINTonE2_affine *p) +{ return (int)(POINTonE2_affine_on_curve(p) | vec_is_zero(p, sizeof(*p))); } + +static bool_t POINTonE2_on_curve(const POINTonE2 *p) +{ + vec384x XXX, YY, BZ6; + limb_t inf = vec_is_zero(p->Z, sizeof(p->Z)); + + sqr_fp2(BZ6, p->Z); + mul_fp2(BZ6, BZ6, p->Z); + sqr_fp2(XXX, BZ6); /* Z^6 */ + mul_by_b_onE2(BZ6, XXX); /* B*Z^6 */ + + sqr_fp2(XXX, p->X); + mul_fp2(XXX, XXX, p->X); /* X^3 */ + add_fp2(XXX, XXX, BZ6); /* X^3 + B*Z^6 */ + + sqr_fp2(YY, p->Y); /* Y^2 */ + + return vec_is_equal(XXX, YY, sizeof(XXX)) | inf; +} + +int blst_p2_on_curve(const POINTonE2 *p) +{ return (int)POINTonE2_on_curve(p); } + +static limb_t POINTonE2_affine_Serialize_BE(unsigned char out[192], + const POINTonE2_affine *in) +{ + vec384x temp; + + from_fp(temp[1], in->X[1]); + be_bytes_from_limbs(out, temp[1], sizeof(temp[1])); + from_fp(temp[0], in->X[0]); + be_bytes_from_limbs(out + 48, temp[0], sizeof(temp[0])); + + from_fp(temp[1], in->Y[1]); + be_bytes_from_limbs(out + 96, temp[1], sizeof(temp[1])); + from_fp(temp[0], in->Y[0]); + be_bytes_from_limbs(out + 144, temp[0], sizeof(temp[0])); + + return sgn0_pty_mod_384x(temp, BLS12_381_P); +} + +void blst_p2_affine_serialize(unsigned char out[192], + const POINTonE2_affine *in) +{ + if (vec_is_zero(in->X, 2*sizeof(in->X))) { + bytes_zero(out, 192); + out[0] = 0x40; /* infinitiy bit */ + } else { + (void)POINTonE2_affine_Serialize_BE(out, in); + } +} + +static limb_t POINTonE2_Serialize_BE(unsigned char out[192], + const POINTonE2 *in) +{ + POINTonE2 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p2, sizeof(in->Z))) { + POINTonE2_from_Jacobian(&p, in); + in = &p; + } + + return POINTonE2_affine_Serialize_BE(out, (const POINTonE2_affine *)in); +} + +static void POINTonE2_Serialize(unsigned char out[192], const POINTonE2 *in) +{ + if (vec_is_zero(in->Z, sizeof(in->Z))) { + bytes_zero(out, 192); + out[0] = 0x40; /* infinitiy bit */ + } else { + (void)POINTonE2_Serialize_BE(out, in); + } +} + +void blst_p2_serialize(unsigned char out[192], const POINTonE2 *in) +{ POINTonE2_Serialize(out, in); } + +static limb_t POINTonE2_affine_Compress_BE(unsigned char out[96], + const POINTonE2_affine *in) +{ + vec384 temp; + + from_fp(temp, in->X[1]); + be_bytes_from_limbs(out, temp, sizeof(temp)); + from_fp(temp, in->X[0]); + be_bytes_from_limbs(out + 48, temp, sizeof(temp)); + + return sgn0_pty_mont_384x(in->Y, BLS12_381_P, p0); +} + +void blst_p2_affine_compress(unsigned char out[96], const POINTonE2_affine *in) +{ + if (vec_is_zero(in->X, 2*sizeof(in->X))) { + bytes_zero(out, 96); + out[0] = 0xc0; /* compressed and infinitiy bits */ + } else { + limb_t sign = POINTonE2_affine_Compress_BE(out, in); + out[0] |= (unsigned char)(0x80 | ((sign & 2) << 4)); + } +} + +static limb_t POINTonE2_Compress_BE(unsigned char out[96], + const POINTonE2 *in) +{ + POINTonE2 p; + + if (!vec_is_equal(in->Z, BLS12_381_Rx.p, sizeof(in->Z))) { + POINTonE2_from_Jacobian(&p, in); + in = &p; + } + + return POINTonE2_affine_Compress_BE(out, (const POINTonE2_affine *)in); +} + +void blst_p2_compress(unsigned char out[96], const POINTonE2 *in) +{ + if (vec_is_zero(in->Z, sizeof(in->Z))) { + bytes_zero(out, 96); + out[0] = 0xc0; /* compressed and infinitiy bits */ + } else { + limb_t sign = POINTonE2_Compress_BE(out, in); + out[0] |= (unsigned char)(0x80 | ((sign & 2) << 4)); + } +} + +static limb_t POINTonE2_Uncompress_BE(POINTonE2_affine *out, + const unsigned char in[96]) +{ + POINTonE2_affine ret; + vec384 temp; + + limbs_from_be_bytes(ret.X[1], in, sizeof(ret.X[1])); + limbs_from_be_bytes(ret.X[0], in + 48, sizeof(ret.X[0])); + + /* clear top 3 bits in case caller was conveying some information there */ + ret.X[1][sizeof(ret.X[1])/sizeof(limb_t)-1] &= ((limb_t)0-1) >> 3; + add_fp(temp, ret.X[1], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X[1], sizeof(temp))) + return (limb_t)0 - BLST_BAD_ENCODING; + + add_fp(temp, ret.X[0], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X[0], sizeof(temp))) + return (limb_t)0 - BLST_BAD_ENCODING; + + mul_fp(ret.X[0], ret.X[0], BLS12_381_RR); + mul_fp(ret.X[1], ret.X[1], BLS12_381_RR); + + sqr_fp2(ret.Y, ret.X); + mul_fp2(ret.Y, ret.Y, ret.X); + add_fp2(ret.Y, ret.Y, B_E2); /* X^3 + B */ + if (!sqrt_fp2(ret.Y, ret.Y)) + return (limb_t)0 - BLST_POINT_NOT_ON_CURVE; + + vec_copy(out, &ret, sizeof(ret)); + + return sgn0_pty_mont_384x(out->Y, BLS12_381_P, p0); +} + +static BLST_ERROR POINTonE2_Uncompress_Z(POINTonE2_affine *out, + const unsigned char in[96]) +{ + unsigned char in0 = in[0]; + limb_t sgn0_pty; + + if ((in0 & 0x80) == 0) /* compressed bit */ + return BLST_BAD_ENCODING; + + if (in0 & 0x40) { /* infinity bit */ + if (byte_is_zero(in0 & 0x3f) & bytes_are_zero(in+1, 95)) { + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } else { + return BLST_BAD_ENCODING; + } + } + + sgn0_pty = POINTonE2_Uncompress_BE(out, in); + + if (sgn0_pty > 3) + return (BLST_ERROR)(0 - sgn0_pty); /* POINT_NOT_ON_CURVE */ + + sgn0_pty >>= 1; /* skip over parity bit */ + sgn0_pty ^= (in0 & 0x20) >> 5; + cneg_fp2(out->Y, out->Y, sgn0_pty); + + return BLST_SUCCESS; +} + +BLST_ERROR blst_p2_uncompress(POINTonE2_affine *out, const unsigned char in[96]) +{ return POINTonE2_Uncompress_Z(out, in); } + +static BLST_ERROR POINTonE2_Deserialize_BE(POINTonE2_affine *out, + const unsigned char in[192]) +{ + POINTonE2_affine ret; + vec384 temp; + + limbs_from_be_bytes(ret.X[1], in, sizeof(ret.X[1])); + limbs_from_be_bytes(ret.X[0], in + 48, sizeof(ret.X[0])); + limbs_from_be_bytes(ret.Y[1], in + 96, sizeof(ret.Y[1])); + limbs_from_be_bytes(ret.Y[0], in + 144, sizeof(ret.Y[0])); + + /* clear top 3 bits in case caller was conveying some information there */ + ret.X[1][sizeof(ret.X[1])/sizeof(limb_t)-1] &= ((limb_t)0-1) >> 3; + add_fp(temp, ret.X[1], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X[1], sizeof(temp))) + return BLST_BAD_ENCODING; + + add_fp(temp, ret.X[0], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.X[0], sizeof(temp))) + return BLST_BAD_ENCODING; + + add_fp(temp, ret.Y[1], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.Y[1], sizeof(temp))) + return BLST_BAD_ENCODING; + + add_fp(temp, ret.Y[0], ZERO_384); /* less than modulus? */ + if (!vec_is_equal(temp, ret.Y[0], sizeof(temp))) + return BLST_BAD_ENCODING; + + mul_fp(ret.X[0], ret.X[0], BLS12_381_RR); + mul_fp(ret.X[1], ret.X[1], BLS12_381_RR); + mul_fp(ret.Y[0], ret.Y[0], BLS12_381_RR); + mul_fp(ret.Y[1], ret.Y[1], BLS12_381_RR); + + if (!POINTonE2_affine_on_curve(&ret)) + return BLST_POINT_NOT_ON_CURVE; + + vec_copy(out, &ret, sizeof(ret)); + + return BLST_SUCCESS; +} + +static BLST_ERROR POINTonE2_Deserialize_Z(POINTonE2_affine *out, + const unsigned char in[192]) +{ + unsigned char in0 = in[0]; + + if ((in0 & 0xe0) == 0) + return POINTonE2_Deserialize_BE(out, in); + + if (in0 & 0x80) /* compressed bit */ + return POINTonE2_Uncompress_Z(out, in); + + if (in0 & 0x40) { /* infinity bit */ + if (byte_is_zero(in0 & 0x3f) & bytes_are_zero(in+1, 191)) { + vec_zero(out, sizeof(*out)); + return BLST_SUCCESS; + } + } + + return BLST_BAD_ENCODING; +} + +BLST_ERROR blst_p2_deserialize(POINTonE2_affine *out, + const unsigned char in[192]) +{ return POINTonE2_Deserialize_Z(out, in); } + +#include "ec_ops.h" +POINT_DADD_IMPL(POINTonE2, 384x, fp2) +POINT_DADD_AFFINE_IMPL_A0(POINTonE2, 384x, fp2, BLS12_381_Rx.p2) +POINT_ADD_IMPL(POINTonE2, 384x, fp2) +POINT_ADD_AFFINE_IMPL(POINTonE2, 384x, fp2, BLS12_381_Rx.p2) +POINT_DOUBLE_IMPL_A0(POINTonE2, 384x, fp2) +POINT_IS_EQUAL_IMPL(POINTonE2, 384x, fp2) + +void blst_p2_add(POINTonE2 *out, const POINTonE2 *a, const POINTonE2 *b) +{ POINTonE2_add(out, a, b); } + +void blst_p2_add_or_double(POINTonE2 *out, const POINTonE2 *a, + const POINTonE2 *b) +{ POINTonE2_dadd(out, a, b, NULL); } + +void blst_p2_add_affine(POINTonE2 *out, const POINTonE2 *a, + const POINTonE2_affine *b) +{ POINTonE2_add_affine(out, a, b); } + +void blst_p2_add_or_double_affine(POINTonE2 *out, const POINTonE2 *a, + const POINTonE2_affine *b) +{ POINTonE2_dadd_affine(out, a, b); } + +void blst_p2_double(POINTonE2 *out, const POINTonE2 *a) +{ POINTonE2_double(out, a); } + +int blst_p2_is_equal(const POINTonE2 *a, const POINTonE2 *b) +{ return (int)POINTonE2_is_equal(a, b); } + +#include "ec_mult.h" +POINT_MULT_SCALAR_WX_IMPL(POINTonE2, 4) +POINT_MULT_SCALAR_WX_IMPL(POINTonE2, 5) + +#ifdef __BLST_PRIVATE_TESTMODE__ +POINT_AFFINE_MULT_SCALAR_IMPL(POINTonE2) + +DECLARE_PRIVATE_POINTXZ(POINTonE2, 384x) +POINT_LADDER_PRE_IMPL(POINTonE2, 384x, fp2) +POINT_LADDER_STEP_IMPL_A0(POINTonE2, 384x, fp2, onE2) +POINT_LADDER_POST_IMPL_A0(POINTonE2, 384x, fp2, onE2) +POINT_MULT_SCALAR_LADDER_IMPL(POINTonE2) +#endif + +static void psi(POINTonE2 *out, const POINTonE2 *in) +{ + static const vec384x frobenius_x = { /* 1/(1 + i)^((P-1)/3) */ + { 0 }, + { /* (0x1a0111ea397fe699ec02408663d4de85aa0d857d89759ad4 + 897d29650fb85f9b409427eb4f49fffd8bfd00000000aaad << 384) % P */ + TO_LIMB_T(0x890dc9e4867545c3), TO_LIMB_T(0x2af322533285a5d5), + TO_LIMB_T(0x50880866309b7e2c), TO_LIMB_T(0xa20d1b8c7e881024), + TO_LIMB_T(0x14e4f04fe2db9068), TO_LIMB_T(0x14e56d3f1564853a) } + }; + static const vec384x frobenius_y = { /* 1/(1 + i)^((P-1)/2) */ + { /* (0x135203e60180a68ee2e9c448d77a2cd91c3dedd930b1cf60 + ef396489f61eb45e304466cf3e67fa0af1ee7b04121bdea2 << 384) % P */ + TO_LIMB_T(0x3e2f585da55c9ad1), TO_LIMB_T(0x4294213d86c18183), + TO_LIMB_T(0x382844c88b623732), TO_LIMB_T(0x92ad2afd19103e18), + TO_LIMB_T(0x1d794e4fac7cf0b9), TO_LIMB_T(0x0bd592fc7d825ec8) }, + { /* (0x06af0e0437ff400b6831e36d6bd17ffe48395dabc2d3435e + 77f76e17009241c5ee67992f72ec05f4c81084fbede3cc09 << 384) % P */ + TO_LIMB_T(0x7bcfa7a25aa30fda), TO_LIMB_T(0xdc17dec12a927e7c), + TO_LIMB_T(0x2f088dd86b4ebef1), TO_LIMB_T(0xd1ca2087da74d4a7), + TO_LIMB_T(0x2da2596696cebc1d), TO_LIMB_T(0x0e2b7eedbbfd87d2) }, + }; + + vec_copy(out, in, sizeof(*out)); + cneg_fp(out->X[1], out->X[1], 1); mul_fp2(out->X, out->X, frobenius_x); + cneg_fp(out->Y[1], out->Y[1], 1); mul_fp2(out->Y, out->Y, frobenius_y); + cneg_fp(out->Z[1], out->Z[1], 1); +} + +/* Galbraith-Lin-Scott, ~67% faster than POINTonE2_mul_w5 */ +static void POINTonE2_mult_gls(POINTonE2 *out, const POINTonE2 *in, + const pow256 SK) +{ + union { vec256 l; pow256 s; } val; + + /* break down SK to "digits" with |z| as radix [in constant time] */ + + limbs_from_le_bytes(val.l, SK, 32); + div_by_zz(val.l); + div_by_z(val.l); + div_by_z(val.l + NLIMBS(256)/2); + le_bytes_from_limbs(val.s, val.l, 32); + + { + const byte *scalars[2] = { val.s, NULL }; + POINTonE2 table[4][1<<(5-1)]; /* 18KB */ + size_t i; + + POINTonE2_precompute_w5(table[0], in); + for (i = 0; i < 1<<(5-1); i++) { + psi(&table[1][i], &table[0][i]); + psi(&table[2][i], &table[1][i]); + psi(&table[3][i], &table[2][i]); + POINTonE2_cneg(&table[1][i], 1); /* account for z being negative */ + POINTonE2_cneg(&table[3][i], 1); + } + + POINTonE2s_mult_w5(out, NULL, 4, scalars, 64, table); + } + + vec_zero(val.l, sizeof(val)); /* scrub the copy of SK */ +} + +static void POINTonE2_sign(POINTonE2 *out, const POINTonE2 *in, const pow256 SK) +{ + vec384x Z, ZZ; + limb_t inf; + + POINTonE2_mult_gls(out, in, SK); + + /* convert to affine to remove possible bias in out->Z */ + inf = vec_is_zero(out->Z, sizeof(out->Z)); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + flt_reciprocal_fp2(Z, out->Z); /* 1/Z */ +#else + reciprocal_fp2(Z, out->Z); /* 1/Z */ +#endif + + sqr_fp2(ZZ, Z); + mul_fp2(out->X, out->X, ZZ); /* X = X/Z^2 */ + + mul_fp2(ZZ, ZZ, Z); + mul_fp2(out->Y, out->Y, ZZ); /* Y = Y/Z^3 */ + + vec_select(out->Z, out->Z, BLS12_381_G2.Z, sizeof(BLS12_381_G2.Z), + inf); /* Z = inf ? 0 : 1 */ +} + +void blst_sk_to_pk_in_g2(POINTonE2 *out, const pow256 SK) +{ POINTonE2_sign(out, &BLS12_381_G2, SK); } + +void blst_sign_pk_in_g1(POINTonE2 *out, const POINTonE2 *msg, const pow256 SK) +{ POINTonE2_sign(out, msg, SK); } + +void blst_sk_to_pk2_in_g2(unsigned char out[192], POINTonE2_affine *PK, + const pow256 SK) +{ + POINTonE2 P[1]; + + POINTonE2_sign(P, &BLS12_381_G2, SK); + if (PK != NULL) + vec_copy(PK, P, sizeof(*PK)); + if (out != NULL) { + limb_t sgn0_pty = POINTonE2_Serialize_BE(out, P); + out[0] |= (sgn0_pty & 2) << 4; /* pre-decorate */ + out[0] |= vec_is_zero(P->Z, sizeof(P->Z)) << 6; + } +} + +void blst_sign_pk2_in_g1(unsigned char out[192], POINTonE2_affine *sig, + const POINTonE2 *hash, const pow256 SK) +{ + POINTonE2 P[1]; + + POINTonE2_sign(P, hash, SK); + if (sig != NULL) + vec_copy(sig, P, sizeof(*sig)); + if (out != NULL) { + limb_t sgn0_pty = POINTonE2_Serialize_BE(out, P); + out[0] |= (sgn0_pty & 2) << 4; /* pre-decorate */ + out[0] |= vec_is_zero(P->Z, sizeof(P->Z)) << 6; + } +} + +void blst_p2_mult(POINTonE2 *out, const POINTonE2 *a, + const byte *scalar, size_t nbits) +{ + if (nbits < 144) { + if (nbits) + POINTonE2_mult_w4(out, a, scalar, nbits); + else + vec_zero(out, sizeof(*out)); + } else if (nbits <= 256) { + union { vec256 l; pow256 s; } val; + size_t i, j, top, mask = (size_t)0 - 1; + + /* this is not about constant-time-ness, but branch optimization */ + for (top = (nbits + 7)/8, i=0, j=0; i<sizeof(val.s);) { + val.s[i++] = scalar[j] & mask; + mask = 0 - ((i - top) >> (8*sizeof(top)-1)); + j += 1 & mask; + } + + if (check_mod_256(val.s, BLS12_381_r)) /* z^4 is the formal limit */ + POINTonE2_mult_gls(out, a, val.s); + else /* should never be the case, added for formal completeness */ + POINTonE2_mult_w5(out, a, scalar, nbits); + + vec_zero(val.l, sizeof(val)); + } else { /* should never be the case, added for formal completeness */ + POINTonE2_mult_w5(out, a, scalar, nbits); + } +} + +void blst_p2_unchecked_mult(POINTonE2 *out, const POINTonE2 *a, + const byte *scalar, size_t nbits) +{ + if (nbits) + POINTonE2_mult_w4(out, a, scalar, nbits); + else + vec_zero(out, sizeof(*out)); +} + +int blst_p2_affine_is_equal(const POINTonE2_affine *a, + const POINTonE2_affine *b) +{ return (int)vec_is_equal(a, b, sizeof(*a)); } + +int blst_p2_is_inf(const POINTonE2 *p) +{ return (int)vec_is_zero(p->Z, sizeof(p->Z)); } + +const POINTonE2 *blst_p2_generator(void) +{ return &BLS12_381_G2; } + +int blst_p2_affine_is_inf(const POINTonE2_affine *p) +{ return (int)vec_is_zero(p, sizeof(*p)); } + +const POINTonE2_affine *blst_p2_affine_generator(void) +{ return (const POINTonE2_affine *)&BLS12_381_G2; } + +size_t blst_p2_sizeof(void) +{ return sizeof(POINTonE2); } + +size_t blst_p2_affine_sizeof(void) +{ return sizeof(POINTonE2_affine); } diff --git a/crypto/blst_src/ec_mult.h b/crypto/blst_src/ec_mult.h new file mode 100644 index 00000000000..192f7337cbf --- /dev/null +++ b/crypto/blst_src/ec_mult.h @@ -0,0 +1,289 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_EC_MULT_H__ +#define __BLS12_381_ASM_EC_MULT_H__ + +#include "point.h" + +/* Works up to 9 bits */ +static limb_t get_wval(const byte *d, size_t off, size_t bits) +{ + size_t top = off + bits - 1; + limb_t ret; + + ret = ((limb_t)d[top / 8] << 8) | d[off / 8]; + + return ret >> (off%8); +} + +/* Works up to 25 bits. */ +static limb_t get_wval_limb(const byte *d, size_t off, size_t bits) +{ + size_t i, top = (off + bits - 1)/8; + limb_t ret, mask = (limb_t)0 - 1; + + d += off/8; + top -= off/8-1; + + /* this is not about constant-time-ness, but branch optimization */ + for (ret=0, i=0; i<4;) { + ret |= (*d & mask) << (8*i); + mask = (limb_t)0 - ((++i - top) >> (8*sizeof(top)-1)); + d += 1 & mask; + } + + return ret >> (off%8); +} + +/* + * Window value encoding that utilizes the fact that -P is trivially + * calculated, which allows to halve the size of pre-computed table, + * is attributed to A. D. Booth, hence the name of the subroutines... + */ +static limb_t booth_encode(limb_t wval, size_t sz) +{ + limb_t mask = 0 - (wval >> sz); /* "sign" bit -> mask */ + + wval = (wval + 1) >> 1; + wval = (wval & ~mask) | ((0-wval) & mask); + + /* &0x1f, but <=0x10, is index in table, rest is extended "sign" bit */ + return wval; +} + +/* + * Key feature of these constant-time subroutines is that they tolerate + * zeros in most significant bit positions of the scalar[s], or in other + * words, zero-padded scalar values. This means that one can and should + * pass order's bit-length, which is customarily publicly known, instead + * of the factual scalars' bit-lengths. This is facilitated by point + * addition subroutines implemented to handle points at infinity, which + * are encoded as Z==0. [Doubling agorithms handle such points at + * infinity "naturally," since resulting Z is product of original Z.] + */ +#define POINT_MULT_SCALAR_WX_IMPL(ptype, SZ) \ +static void ptype##_gather_booth_w##SZ(ptype *restrict p, \ + const ptype table[1<<(SZ-1)], \ + limb_t booth_idx) \ +{ \ + size_t i; \ + bool_t booth_sign = (booth_idx >> SZ) & 1; \ +\ + booth_idx &= (1<<SZ) - 1; \ + vec_zero(p, sizeof(ptype)); /* implicit infinity at table[-1] */\ + /* ~6% with -Os, ~2% with -O3 ... */\ + for (i = 1; i <= 1<<(SZ-1); i++) \ + ptype##_ccopy(p, table + i - 1, byte_is_zero((byte)(i ^ booth_idx))); \ +\ + ptype##_cneg(p, booth_sign); \ +} \ +\ +static void ptype##_precompute_w##SZ(ptype row[], const ptype *point) \ +{ \ + size_t i, j; \ + /* row[-1] is implicit infinity */\ + vec_copy(&row[0], point, sizeof(ptype)); /* row[0]=p*1 */\ + ptype##_double(&row[1], point); /* row[1]=p*(1+1) */\ + for (i = 2, j = 1; i < 1<<(SZ-1); i += 2, j++) \ + ptype##_add(&row[i], &row[j], &row[j-1]), /* row[2]=p*(2+1) */\ + ptype##_double(&row[i+1], &row[j]); /* row[3]=p*(2+2) */\ +} /* row[4] ... */\ +\ +static void ptype##s_mult_w##SZ(ptype *ret, \ + const ptype *points[], size_t npoints, \ + const byte *scalars[], size_t bits, \ + ptype table[][1<<(SZ-1)]) \ +{ \ + limb_t wmask, wval; \ + size_t i, j, window, nbytes; \ + const byte *scalar, **scalar_s = scalars; \ + ptype temp[1]; \ +\ + if (table == NULL) \ + table = (ptype (*)[1<<(SZ-1)])alloca((1<<(SZ-1)) * sizeof(ptype) * \ + npoints); \ +\ + if (points != NULL) { \ + const ptype *point = NULL; \ + for (i = 0; i < npoints; i++) \ + point = *points ? *points++ : point+1, \ + ptype##_precompute_w##SZ(table[i], point); \ + } \ +\ + nbytes = (bits + 7)/8; /* convert |bits| to bytes */ \ + scalar = *scalar_s++; \ +\ + /* top excess bits modulo target window size */ \ + window = bits % SZ; /* yes, it may be zero */ \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ +\ + bits -= window; \ + if (bits > 0) \ + wval = get_wval(scalar, bits - 1, window + 1) & wmask; \ + else \ + wval = (scalar[0] << 1) & wmask; \ +\ + wval = booth_encode(wval, SZ); \ + ptype##_gather_booth_w##SZ(ret, table[0], wval); \ +\ + i = 1; \ + while (bits > 0) { \ + for (; i < npoints; i++) { \ + scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ + wval = get_wval(scalar, bits - 1, window + 1) & wmask; \ + wval = booth_encode(wval, SZ); \ + ptype##_gather_booth_w##SZ(temp, table[i], wval); \ + ptype##_dadd(ret, ret, temp, NULL); \ + } \ +\ + for (j = 0; j < SZ; j++) \ + ptype##_double(ret, ret); \ +\ + window = SZ; \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ + bits -= window; \ + i = 0; scalar_s = scalars; \ + } \ +\ + for (; i < npoints; i++) { \ + scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ + wval = (scalar[0] << 1) & wmask; \ + wval = booth_encode(wval, SZ); \ + ptype##_gather_booth_w##SZ(temp, table[i], wval); \ + ptype##_dadd(ret, ret, temp, NULL); \ + } \ +} \ +\ +static void ptype##_mult_w##SZ(ptype *ret, const ptype *point, \ + const byte *scalar, size_t bits) \ +{ \ + limb_t wmask, wval; \ + size_t j, window; \ + ptype temp[1]; \ + ptype table[1<<(SZ-1)]; \ +\ + ptype##_precompute_w##SZ(table, point); \ +\ + /* top excess bits modulo target window size */ \ + window = bits % SZ; /* yes, it may be zero */ \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ +\ + bits -= window; \ + wval = bits ? get_wval(scalar, bits - 1, window + 1) \ + : (limb_t)scalar[0] << 1; \ + wval &= wmask; \ + wval = booth_encode(wval, SZ); \ + ptype##_gather_booth_w##SZ(ret, table, wval); \ +\ + while (bits > 0) { \ + for (j = 0; j < SZ; j++) \ + ptype##_double(ret, ret); \ +\ + window = SZ; \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ + bits -= window; \ +\ + wval = bits ? get_wval(scalar, bits - 1, window + 1) \ + : (limb_t)scalar[0] << 1; \ + wval &= wmask; \ + wval = booth_encode(wval, SZ); \ + ptype##_gather_booth_w##SZ(temp, table, wval); \ + if (bits > 0) ptype##_add(ret, ret, temp); \ + else ptype##_dadd(ret, ret, temp, NULL); \ + } \ +} + +#if 0 +/* ~50%, or ~2x[!] slower than w5... */ +#define POINT_MULT_SCALAR_LADDER_IMPL(ptype) \ +static void ptype##_mult_ladder(ptype *ret, const ptype *p, \ + const byte *scalar, size_t bits) \ +{ \ + ptype sum[1]; \ + bool_t bit, pbit = 0; \ +\ + vec_copy(sum, p, sizeof(ptype)); \ + vec_zero(ret, sizeof(ptype)); /* infinity */ \ +\ + while (bits--) { \ + bit = is_bit_set(scalar, bits); \ + bit ^= pbit; \ + ptype##_cswap(ret, sum, bit); \ + ptype##_add(sum, sum, ret); \ + ptype##_double(ret, ret); \ + pbit ^= bit; \ + } \ + ptype##_cswap(ret, sum, pbit); \ +} +#else +/* >40% better performance than above, [and ~30% slower than w5]... */ +#define POINT_MULT_SCALAR_LADDER_IMPL(ptype) \ +static void ptype##_mult_ladder(ptype *out, const ptype *p, \ + const byte *scalar, size_t bits) \ +{ \ + ptype##xz sum[1]; \ + ptype##xz pxz[1]; \ + ptype##xz ret[1]; \ + bool_t bit, pbit = 0; \ +\ + ptype##xz_ladder_pre(pxz, p); \ + vec_copy(sum, pxz, sizeof(ptype##xz)); \ + vec_zero(ret, sizeof(ptype##xz)); /* infinity */ \ +\ + while (bits--) { \ + bit = is_bit_set(scalar, bits); \ + bit ^= pbit; \ + ptype##xz_cswap(ret, sum, bit); \ + ptype##xz_ladder_step(ret, sum, pxz); \ + pbit ^= bit; \ + } \ + ptype##xz_cswap(ret, sum, pbit); \ + ptype##xz_ladder_post(out, ret, sum, pxz, p->Y); \ +} +#endif + +/* + * Sole reason for existence of this implementation is that addition + * with affine point renders a share of multiplications redundant by + * virtue of Z==1. And since pre-defined generator point can be and + * customarily is instantiated affine, it would be hardly appropriate + * to pass on this opportunity. Though while it's faster than the + * generic ladder implementation, by ~25%, it's not faster than XZ one + * above, <15% slower. Just in case, it's faster than generic ladder + * even if one accounts for prior conversion to affine coordinates, + * so that choice [for resource-constrained case] is actually between + * this plus said conversion and XZ ladder... + * + * To summarize, if ptype##_mult_w5 executed in one unit of time, then + * - naive ptype##_mult_ladder would execute in ~2; + * - XZ version above - in ~1.4; + * - ptype##_affine_mult_ladder below - in ~1.65; + * - [small-footprint ptype##_to_affine would run in ~0.18]. + * + * Caveat lector, |p_affine|*(order+2) produces wrong result, because + * addition doesn't handle doubling. Indeed, P*(order+1) is P and it + * fails to add with itself producing infinity in last addition. But + * as long as |scalar| is reduced modulo order, as it should be, it's + * not a problem... + */ +#define POINT_AFFINE_MULT_SCALAR_IMPL(ptype) \ +static void ptype##_affine_mult_ladder(ptype *ret, \ + const ptype##_affine *p_affine, \ + const byte *scalar, size_t bits) \ +{ \ + ptype sum[1]; \ + bool_t bit; \ +\ + vec_zero(ret, sizeof(ptype)); /* infinity */ \ +\ + while (bits--) { \ + ptype##_double(ret, ret); \ + ptype##_add_affine(sum, ret, p_affine); \ + bit = (scalar[bits / LIMB_T_BITS] >> (bits % LIMB_T_BITS)) & 1; \ + ptype##_ccopy(ret, sum, bit); \ + } \ +} +#endif diff --git a/crypto/blst_src/ec_ops.h b/crypto/blst_src/ec_ops.h new file mode 100644 index 00000000000..0d531f816e2 --- /dev/null +++ b/crypto/blst_src/ec_ops.h @@ -0,0 +1,787 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_384_ASM_EC_OPS_H__ +#define __BLS12_384_ASM_EC_OPS_H__ +/* + * Addition that can handle doubling [as well as points at infinity, + * which are encoded as Z==0] in constant time. It naturally comes at + * cost, but this subroutine should be called only when independent + * points are processed, which is considered reasonable compromise. + * For example, ptype##s_mult_w5 calls it, but since *major* gain is + * result of pure doublings being effectively divided by amount of + * points, slightly slower addition can be tolerated. But what is the + * additional cost more specifically? Best addition result is 11M+5S, + * while this routine takes 13M+5S (+1M+1S if a4!=0), as per + * + * -------------+------------- + * addition | doubling + * -------------+------------- + * U1 = X1*Z2^2 | U1 = X1 + * U2 = X2*Z1^2 | + * S1 = Y1*Z2^3 | S1 = Y1 + * S2 = Y2*Z1^3 | + * zz = Z1*Z2 | zz = Z1 + * H = U2-U1 | H' = 2*Y1 + * R = S2-S1 | R' = 3*X1^2[+a*Z1^4] + * sx = U1+U2 | sx = X1+X1 + * -------------+------------- + * H!=0 || R!=0 | H==0 && R==0 + * + * X3 = R^2-H^2*sx + * Y3 = R*(H^2*U1-X3)-H^3*S1 + * Z3 = H*zz + * + * As for R!=0 condition in context of H==0, a.k.a. P-P. The result is + * infinity by virtue of Z3 = (U2-U1)*zz = H*zz = 0*zz == 0. + */ +#define POINT_DADD_IMPL(ptype, bits, field) \ +static void ptype##_dadd(ptype *out, const ptype *p1, const ptype *p2, \ + const vec##bits a4) \ +{ \ + ptype p3; /* starts as (U1, S1, zz) from addition side */\ + struct { vec##bits H, R, sx; } add, dbl; \ + bool_t p1inf, p2inf, is_dbl; \ +\ + add_##field(dbl.sx, p1->X, p1->X); /* sx = X1+X1 */\ + sqr_##field(dbl.R, p1->X); /* X1^2 */\ + mul_by_3_##field(dbl.R, dbl.R); /* R = 3*X1^2 */\ + add_##field(dbl.H, p1->Y, p1->Y); /* H = 2*Y1 */\ +\ + p2inf = vec_is_zero(p2->Z, sizeof(p2->Z)); \ + sqr_##field(p3.X, p2->Z); /* Z2^2 */\ + mul_##field(p3.Z, p1->Z, p2->Z); /* Z1*Z2 */\ + p1inf = vec_is_zero(p1->Z, sizeof(p1->Z)); \ + sqr_##field(add.H, p1->Z); /* Z1^2 */\ +\ + if (a4 != NULL) { \ + sqr_##field(p3.Y, add.H); /* Z1^4, [borrow p3.Y] */\ + mul_##field(p3.Y, p3.Y, a4); \ + add_##field(dbl.R, dbl.R, p3.Y);/* R = 3*X1^2+a*Z1^4 */\ + } \ +\ + mul_##field(p3.Y, p1->Y, p2->Z); \ + mul_##field(p3.Y, p3.Y, p3.X); /* S1 = Y1*Z2^3 */\ + mul_##field(add.R, p2->Y, p1->Z); \ + mul_##field(add.R, add.R, add.H); /* S2 = Y2*Z1^3 */\ + sub_##field(add.R, add.R, p3.Y); /* R = S2-S1 */\ +\ + mul_##field(p3.X, p3.X, p1->X); /* U1 = X1*Z2^2 */\ + mul_##field(add.H, add.H, p2->X); /* U2 = X2*Z1^2 */\ +\ + add_##field(add.sx, add.H, p3.X); /* sx = U1+U2 */\ + sub_##field(add.H, add.H, p3.X); /* H = U2-U1 */\ +\ + /* make the choice between addition and doubling */\ + is_dbl = vec_is_zero(add.H, 2*sizeof(add.H)); \ + vec_select(&p3, p1, &p3, sizeof(p3), is_dbl); \ + vec_select(&add, &dbl, &add, sizeof(add), is_dbl); \ + /* |p3| and |add| hold all inputs now, |p3| will hold output */\ +\ + mul_##field(p3.Z, p3.Z, add.H); /* Z3 = H*Z1*Z2 */\ +\ + sqr_##field(dbl.H, add.H); /* H^2 */\ + mul_##field(dbl.R, dbl.H, add.H); /* H^3 */\ + mul_##field(dbl.R, dbl.R, p3.Y); /* H^3*S1 */\ + mul_##field(p3.Y, dbl.H, p3.X); /* H^2*U1 */\ +\ + mul_##field(dbl.H, dbl.H, add.sx); /* H^2*sx */\ + sqr_##field(p3.X, add.R); /* R^2 */\ + sub_##field(p3.X, p3.X, dbl.H); /* X3 = R^2-H^2*sx */\ +\ + sub_##field(p3.Y, p3.Y, p3.X); /* H^2*U1-X3 */\ + mul_##field(p3.Y, p3.Y, add.R); /* R*(H^2*U1-X3) */\ + sub_##field(p3.Y, p3.Y, dbl.R); /* Y3 = R*(H^2*U1-X3)-H^3*S1 */\ +\ + vec_select(&p3, p1, &p3, sizeof(ptype), p2inf); \ + vec_select(out, p2, &p3, sizeof(ptype), p1inf); \ +} + +/* + * Addition with affine point that can handle doubling [as well as + * points at infinity, with |p1| being encoded as Z==0 and |p2| as + * X,Y==0] in constant time. But at what additional cost? Best + * addition result is 7M+4S, while this routine takes 8M+5S, as per + * + * -------------+------------- + * addition | doubling + * -------------+------------- + * U1 = X1 | U1 = X2 + * U2 = X2*Z1^2 | + * S1 = Y1 | S1 = Y2 + * S2 = Y2*Z1^3 | + * H = U2-X1 | H' = 2*Y2 + * R = S2-Y1 | R' = 3*X2^2[+a] + * sx = X1+U2 | sx = X2+X2 + * zz = H*Z1 | zz = H' + * -------------+------------- + * H!=0 || R!=0 | H==0 && R==0 + * + * X3 = R^2-H^2*sx + * Y3 = R*(H^2*U1-X3)-H^3*S1 + * Z3 = zz + * + * As for R!=0 condition in context of H==0, a.k.a. P-P. The result is + * infinity by virtue of Z3 = (U2-U1)*zz = H*zz = 0*zz == 0. + */ +#define POINT_DADD_AFFINE_IMPL_A0(ptype, bits, field, one) \ +static void ptype##_dadd_affine(ptype *out, const ptype *p1, \ + const ptype##_affine *p2) \ +{ \ + ptype p3; /* starts as (,, H*Z1) from addition side */\ + struct { vec##bits H, R, sx; } add, dbl; \ + bool_t p1inf, p2inf, is_dbl; \ +\ + p2inf = vec_is_zero(p2->X, 2*sizeof(p2->X)); \ + add_##field(dbl.sx, p2->X, p2->X); /* sx = X2+X2 */\ + sqr_##field(dbl.R, p2->X); /* X2^2 */\ + mul_by_3_##field(dbl.R, dbl.R); /* R = 3*X2^2 */\ + add_##field(dbl.H, p2->Y, p2->Y); /* H = 2*Y2 */\ +\ + p1inf = vec_is_zero(p1->Z, sizeof(p1->Z)); \ + sqr_##field(add.H, p1->Z); /* Z1^2 */\ + mul_##field(add.R, add.H, p1->Z); /* Z1^3 */\ + mul_##field(add.R, add.R, p2->Y); /* S2 = Y2*Z1^3 */\ + sub_##field(add.R, add.R, p1->Y); /* R = S2-Y1 */\ +\ + mul_##field(add.H, add.H, p2->X); /* U2 = X2*Z1^2 */\ +\ + add_##field(add.sx, add.H, p1->X); /* sx = X1+U2 */\ + sub_##field(add.H, add.H, p1->X); /* H = U2-X1 */\ +\ + mul_##field(p3.Z, add.H, p1->Z); /* Z3 = H*Z1 */\ +\ + /* make the choice between addition and doubling */ \ + is_dbl = vec_is_zero(add.H, 2*sizeof(add.H)); \ + vec_select(p3.X, p2, p1, 2*sizeof(p3.X), is_dbl); \ + vec_select(p3.Z, dbl.H, p3.Z, sizeof(p3.Z), is_dbl);\ + vec_select(&add, &dbl, &add, sizeof(add), is_dbl); \ + /* |p3| and |add| hold all inputs now, |p3| will hold output */\ +\ + sqr_##field(dbl.H, add.H); /* H^2 */\ + mul_##field(dbl.R, dbl.H, add.H); /* H^3 */\ + mul_##field(dbl.R, dbl.R, p3.Y); /* H^3*S1 */\ + mul_##field(p3.Y, dbl.H, p3.X); /* H^2*U1 */\ +\ + mul_##field(dbl.H, dbl.H, add.sx); /* H^2*sx */\ + sqr_##field(p3.X, add.R); /* R^2 */\ + sub_##field(p3.X, p3.X, dbl.H); /* X3 = R^2-H^2*sx */\ +\ + sub_##field(p3.Y, p3.Y, p3.X); /* H^2*U1-X3 */\ + mul_##field(p3.Y, p3.Y, add.R); /* R*(H^2*U1-X3) */\ + sub_##field(p3.Y, p3.Y, dbl.R); /* Y3 = R*(H^2*U1-X3)-H^3*S1 */\ +\ + vec_select(p3.X, p2, p3.X, 2*sizeof(p3.X), p1inf); \ + vec_select(p3.Z, one, p3.Z, sizeof(p3.Z), p1inf); \ + vec_select(out, p1, &p3, sizeof(ptype), p2inf); \ +} + +/* + * https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl + * with twist to handle either input at infinity, which are encoded as Z==0. + */ +#define POINT_ADD_IMPL(ptype, bits, field) \ +static void ptype##_add(ptype *out, const ptype *p1, const ptype *p2) \ +{ \ + ptype p3; \ + vec##bits Z1Z1, Z2Z2, U1, S1, H, I, J; \ + bool_t p1inf, p2inf; \ +\ + p1inf = vec_is_zero(p1->Z, sizeof(p1->Z)); \ + sqr_##field(Z1Z1, p1->Z); /* Z1Z1 = Z1^2 */\ +\ + mul_##field(p3.Z, Z1Z1, p1->Z); /* Z1*Z1Z1 */\ + mul_##field(p3.Z, p3.Z, p2->Y); /* S2 = Y2*Z1*Z1Z1 */\ +\ + p2inf = vec_is_zero(p2->Z, sizeof(p2->Z)); \ + sqr_##field(Z2Z2, p2->Z); /* Z2Z2 = Z2^2 */\ +\ + mul_##field(S1, Z2Z2, p2->Z); /* Z2*Z2Z2 */\ + mul_##field(S1, S1, p1->Y); /* S1 = Y1*Z2*Z2Z2 */\ +\ + sub_##field(p3.Z, p3.Z, S1); /* S2-S1 */\ + add_##field(p3.Z, p3.Z, p3.Z); /* r = 2*(S2-S1) */\ +\ + mul_##field(U1, p1->X, Z2Z2); /* U1 = X1*Z2Z2 */\ + mul_##field(H, p2->X, Z1Z1); /* U2 = X2*Z1Z1 */\ +\ + sub_##field(H, H, U1); /* H = U2-U1 */\ +\ + add_##field(I, H, H); /* 2*H */\ + sqr_##field(I, I); /* I = (2*H)^2 */\ +\ + mul_##field(J, H, I); /* J = H*I */\ + mul_##field(S1, S1, J); /* S1*J */\ +\ + mul_##field(p3.Y, U1, I); /* V = U1*I */\ +\ + sqr_##field(p3.X, p3.Z); /* r^2 */\ + sub_##field(p3.X, p3.X, J); /* r^2-J */\ + sub_##field(p3.X, p3.X, p3.Y); \ + sub_##field(p3.X, p3.X, p3.Y); /* X3 = r^2-J-2*V */\ +\ + sub_##field(p3.Y, p3.Y, p3.X); /* V-X3 */\ + mul_##field(p3.Y, p3.Y, p3.Z); /* r*(V-X3) */\ + sub_##field(p3.Y, p3.Y, S1); \ + sub_##field(p3.Y, p3.Y, S1); /* Y3 = r*(V-X3)-2*S1*J */\ +\ + add_##field(p3.Z, p1->Z, p2->Z); /* Z1+Z2 */\ + sqr_##field(p3.Z, p3.Z); /* (Z1+Z2)^2 */\ + sub_##field(p3.Z, p3.Z, Z1Z1); /* (Z1+Z2)^2-Z1Z1 */\ + sub_##field(p3.Z, p3.Z, Z2Z2); /* (Z1+Z2)^2-Z1Z1-Z2Z2 */\ + mul_##field(p3.Z, p3.Z, H); /* Z3 = ((Z1+Z2)^2-Z1Z1-Z2Z2)*H */\ +\ + vec_select(&p3, p1, &p3, sizeof(ptype), p2inf); \ + vec_select(out, p2, &p3, sizeof(ptype), p1inf); \ +} + +/* + * https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-madd-2007-bl + * with twist to handle either input at infinity, with |p1| encoded as Z==0, + * and |p2| as X==Y==0. + */ +#define POINT_ADD_AFFINE_IMPL(ptype, bits, field, one) \ +static void ptype##_add_affine(ptype *out, const ptype *p1, \ + const ptype##_affine *p2) \ +{ \ + ptype p3; \ + vec##bits Z1Z1, H, HH, I, J; \ + bool_t p1inf, p2inf; \ +\ + p1inf = vec_is_zero(p1->Z, sizeof(p1->Z)); \ +\ + sqr_##field(Z1Z1, p1->Z); /* Z1Z1 = Z1^2 */\ +\ + mul_##field(p3.Z, Z1Z1, p1->Z); /* Z1*Z1Z1 */\ + mul_##field(p3.Z, p3.Z, p2->Y); /* S2 = Y2*Z1*Z1Z1 */\ +\ + p2inf = vec_is_zero(p2->X, 2*sizeof(p2->X)); \ +\ + mul_##field(H, p2->X, Z1Z1); /* U2 = X2*Z1Z1 */\ + sub_##field(H, H, p1->X); /* H = U2-X1 */\ +\ + sqr_##field(HH, H); /* HH = H^2 */\ + add_##field(I, HH, HH); \ + add_##field(I, I, I); /* I = 4*HH */\ +\ + mul_##field(p3.Y, p1->X, I); /* V = X1*I */\ + mul_##field(J, H, I); /* J = H*I */\ + mul_##field(I, J, p1->Y); /* Y1*J */\ +\ + sub_##field(p3.Z, p3.Z, p1->Y); /* S2-Y1 */\ + add_##field(p3.Z, p3.Z, p3.Z); /* r = 2*(S2-Y1) */\ +\ + sqr_##field(p3.X, p3.Z); /* r^2 */\ + sub_##field(p3.X, p3.X, J); /* r^2-J */\ + sub_##field(p3.X, p3.X, p3.Y); \ + sub_##field(p3.X, p3.X, p3.Y); /* X3 = r^2-J-2*V */\ +\ + sub_##field(p3.Y, p3.Y, p3.X); /* V-X3 */\ + mul_##field(p3.Y, p3.Y, p3.Z); /* r*(V-X3) */\ + sub_##field(p3.Y, p3.Y, I); \ + sub_##field(p3.Y, p3.Y, I); /* Y3 = r*(V-X3)-2*Y1*J */\ +\ + add_##field(p3.Z, p1->Z, H); /* Z1+H */\ + sqr_##field(p3.Z, p3.Z); /* (Z1+H)^2 */\ + sub_##field(p3.Z, p3.Z, Z1Z1); /* (Z1+H)^2-Z1Z1 */\ + sub_##field(p3.Z, p3.Z, HH); /* Z3 = (Z1+H)^2-Z1Z1-HH */\ +\ + vec_select(p3.Z, one, p3.Z, sizeof(p3.Z), p1inf); \ + vec_select(p3.X, p2, p3.X, 2*sizeof(p3.X), p1inf); \ + vec_select(out, p1, &p3, sizeof(ptype), p2inf); \ +} + +/* + * https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l + */ +#define POINT_DOUBLE_IMPL_A0(ptype, bits, field) \ +static void ptype##_double(ptype *p3, const ptype *p1) \ +{ \ + vec##bits A, B, C; \ +\ + sqr_##field(A, p1->X); /* A = X1^2 */\ + sqr_##field(B, p1->Y); /* B = Y1^2 */\ + sqr_##field(C, B); /* C = B^2 */\ +\ + add_##field(B, B, p1->X); /* X1+B */\ + sqr_##field(B, B); /* (X1+B)^2 */\ + sub_##field(B, B, A); /* (X1+B)^2-A */\ + sub_##field(B, B, C); /* (X1+B)^2-A-C */\ + add_##field(B, B, B); /* D = 2*((X1+B)^2-A-C) */\ +\ + mul_by_3_##field(A, A); /* E = 3*A */\ +\ + sqr_##field(p3->X, A); /* F = E^2 */\ + sub_##field(p3->X, p3->X, B); \ + sub_##field(p3->X, p3->X, B); /* X3 = F-2*D */\ +\ + add_##field(p3->Z, p1->Z, p1->Z); /* 2*Z1 */\ + mul_##field(p3->Z, p3->Z, p1->Y); /* Z3 = 2*Z1*Y1 */\ +\ + mul_by_8_##field(C, C); /* 8*C */\ + sub_##field(p3->Y, B, p3->X); /* D-X3 */\ + mul_##field(p3->Y, p3->Y, A); /* E*(D-X3) */\ + sub_##field(p3->Y, p3->Y, C); /* Y3 = E*(D-X3)-8*C */\ +} + +#define POINT_LADDER_PRE_IMPL(ptype, bits, field) \ +static void ptype##xz_ladder_pre(ptype##xz *pxz, const ptype *p) \ +{ \ + mul_##field(pxz->X, p->X, p->Z); /* X2 = X1*Z1 */\ + sqr_##field(pxz->Z, p->Z); \ + mul_##field(pxz->Z, pxz->Z, p->Z); /* Z2 = Z1^3 */\ +} + +/* + * https://hyperelliptic.org/EFD/g1p/auto-shortw-xz.html#ladder-ladd-2002-it-3 + * with twist to handle either input at infinity, which are encoded as Z==0. + * Just in case, order of doubling and addition is reverse in comparison to + * hyperelliptic.org entry. This was done to minimize temporary storage. + * + * XZ1 is |p|, XZ2&XZ4 are in&out |r|, XZ3&XZ5 are in&out |s|. + */ +#define POINT_LADDER_STEP_IMPL_A0(ptype, bits, field, suffix4b) \ +static void ptype##xz_ladder_step(ptype##xz *r, ptype##xz *s, \ + const ptype##xz *p) \ +{ \ + ptype##xz p5; \ + vec##bits A, B, C, D, XX, ZZ; \ + bool_t r_inf, s_inf; \ + /* s += r */\ + mul_##field(A, r->X, s->X); /* A = X2*X3 */\ + mul_##field(B, r->Z, s->Z); /* B = Z2*Z3 */\ + mul_##field(C, r->X, s->Z); /* C = X2*Z3 */\ + mul_##field(D, r->Z, s->X); /* D = X3*Z2 */\ +\ + sqr_##field(A, A); /* (A[-a*B])^2 */\ + add_##field(p5.X, C, D); /* C+D */\ + mul_##field(p5.X, p5.X, B); /* B*(C+D) */\ + mul_by_4b_##suffix4b(B, p5.X); /* b4*B*(C+D) */\ + sub_##field(p5.X, A, B); /* (A[-a*B])^2-b4*B*(C+D) */\ + mul_##field(p5.X, p5.X, p->Z); /* X5 = Z1*((A[-a*B])^2-b4*B*(C+D)) */\ +\ + sub_##field(p5.Z, C, D); /* C-D */\ + sqr_##field(p5.Z, p5.Z); /* (C-D)^2 */\ + mul_##field(p5.Z, p5.Z, p->X); /* Z5 = X1*(C-D)^2 */\ +\ + r_inf = vec_is_zero(r->Z, sizeof(r->Z)); \ + s_inf = vec_is_zero(s->Z, sizeof(s->Z)); \ +\ + vec_select(&p5, r, &p5, sizeof(ptype##xz), s_inf); \ + vec_select(s, s, &p5, sizeof(ptype##xz), r_inf); \ + /* r *= 2 */\ + sqr_##field(XX, r->X); /* XX = X2^2 */\ + sqr_##field(ZZ, r->Z); /* ZZ = Z2^2 */\ +\ + add_##field(r->Z, r->X, r->Z); /* X2+Z2 */\ + sqr_##field(r->Z, r->Z); /* (X2+Z2)^2 */\ + sub_##field(r->Z, r->Z, XX); /* (X2+Z2)^2-XX */\ + sub_##field(r->Z, r->Z, ZZ); /* E = (X2+Z2)^2-XX-ZZ */\ +\ + sqr_##field(A, XX); /* (XX[-a*ZZ])^2 */\ + mul_##field(B, r->Z, ZZ); /* E*ZZ */\ + mul_by_4b_##suffix4b(C, B); /* b4*E*ZZ */\ + sub_##field(r->X, A, C); /* X4 = (XX[-a*ZZ])^2-b4*E*ZZ */\ +\ + sqr_##field(ZZ, ZZ); /* ZZ^2 */\ + mul_by_4b_##suffix4b(B, ZZ); /* b4*ZZ^2 */\ + mul_##field(r->Z, r->Z, XX); /* E*(XX[+a*ZZ]) */\ + add_##field(r->Z, r->Z, r->Z); /* 2*E*(XX[+a*ZZ]) */\ + add_##field(r->Z, r->Z, B); /* Z4 = 2*E*(XX[+a*ZZ])+b4*ZZ^2 */\ +} + +/* + * Recover the |r|'s y-coordinate using Eq. (8) from Brier-Joye, + * "Weierstraß Elliptic Curves and Side-Channel Attacks", with XZ twist + * and conversion to Jacobian coordinates from <openssl>/.../ecp_smpl.c, + * and with twist to recover from |s| at infinity [which occurs when + * multiplying by (order-1)]. + * + * X4 = 2*Y1*X2*Z3*Z1*Z2 + * Y4 = 2*b*Z3*(Z1*Z2)^2 + Z3*(a*Z1*Z2+X1*X2)*(X1*Z2+X2*Z1) - X3*(X1*Z2-X2*Z1)^2 + * Z4 = 2*Y1*Z3*Z2^2*Z1 + * + * Z3x2 = 2*Z3 + * Y1Z3x2 = Y1*Z3x2 + * Z1Z2 = Z1*Z2 + * X1Z2 = X1*Z2 + * X2Z1 = X2*Z1 + * X4 = Y1Z3x2*X2*Z1Z2 + * A = b*Z3x2*(Z1Z2)^2 + * B = Z3*(a*Z1Z2+X1*X2)*(X1Z2+X2Z1) + * C = X3*(X1Z2-X2Z1)^2 + * Y4 = A+B-C + * Z4 = Y1Z3x2*Z1Z2*Z2 + * + * XZ1 is |p|, XZ2 is |r|, XZ3 is |s|, 'a' is 0. + */ +#define POINT_LADDER_POST_IMPL_A0(ptype, bits, field, suffixb) \ +static void ptype##xz_ladder_post(ptype *p4, \ + const ptype##xz *r, const ptype##xz *s, \ + const ptype##xz *p, const vec##bits Y1) \ +{ \ + vec##bits Z3x2, Y1Z3x2, Z1Z2, X1Z2, X2Z1, A, B, C; \ + bool_t s_inf; \ +\ + add_##field(Z3x2, s->Z, s->Z); /* Z3x2 = 2*Z3 */\ + mul_##field(Y1Z3x2, Y1, Z3x2); /* Y1Z3x2 = Y1*Z3x2 */\ + mul_##field(Z1Z2, p->Z, r->Z); /* Z1Z2 = Z1*Z2 */\ + mul_##field(X1Z2, p->X, r->Z); /* X1Z2 = X1*Z2 */\ + mul_##field(X2Z1, r->X, p->Z); /* X2Z1 = X2*Z1 */\ +\ + mul_##field(p4->X, Y1Z3x2, r->X); /* Y1Z3x2*X2 */\ + mul_##field(p4->X, p4->X, Z1Z2); /* X4 = Y1Z3x2*X2*Z1Z2 */\ +\ + sqr_##field(A, Z1Z2); /* (Z1Z2)^2 */\ + mul_##field(B, A, Z3x2); /* Z3x2*(Z1Z2)^2 */\ + mul_by_b_##suffixb(A, B); /* A = b*Z3x2*(Z1Z2)^2 */\ +\ + mul_##field(B, p->X, r->X); /* [a*Z1Z2+]X1*X2 */\ + mul_##field(B, B, s->Z); /* Z3*([a*Z1Z2+]X1*X2) */\ + add_##field(C, X1Z2, X2Z1); /* X1Z2+X2Z1 */\ + mul_##field(B, B, C); /* B = Z3*([a*Z2Z1+]X1*X2)*(X1Z2+X2Z1) */\ +\ + sub_##field(C, X1Z2, X2Z1); /* X1Z2-X2Z1 */\ + sqr_##field(C, C); /* (X1Z2-X2Z1)^2 */\ + mul_##field(C, C, s->X); /* C = X3*(X1Z2-X2Z1)^2 */\ +\ + add_##field(A, A, B); /* A+B */\ + sub_##field(A, A, C); /* Y4 = A+B-C */\ +\ + mul_##field(p4->Z, Z1Z2, r->Z); /* Z1Z2*Z2 */\ + mul_##field(p4->Z, p4->Z, Y1Z3x2); /* Y1Z3x2*Z1Z2*Z2 */\ +\ + s_inf = vec_is_zero(s->Z, sizeof(s->Z)); \ + vec_select(p4->X, p->X, p4->X, sizeof(p4->X), s_inf); \ + vec_select(p4->Y, Y1, A, sizeof(p4->Y), s_inf); \ + vec_select(p4->Z, p->Z, p4->Z, sizeof(p4->Z), s_inf); \ + ptype##_cneg(p4, s_inf); \ + /* to Jacobian */\ + mul_##field(p4->X, p4->X, p4->Z); /* X4 = X4*Z4 */\ + sqr_##field(B, p4->Z); \ + mul_##field(p4->Y, p4->Y, B); /* Y4 = Y4*Z4^2 */\ +} + +#define POINT_IS_EQUAL_IMPL(ptype, bits, field) \ +static limb_t ptype##_is_equal(const ptype *p1, const ptype *p2) \ +{ \ + vec##bits Z1Z1, Z2Z2; \ + ptype##_affine a1, a2; \ + bool_t is_inf1 = vec_is_zero(p1->Z, sizeof(p1->Z)); \ + bool_t is_inf2 = vec_is_zero(p2->Z, sizeof(p2->Z)); \ +\ + sqr_##field(Z1Z1, p1->Z); /* Z1Z1 = Z1^2 */\ + sqr_##field(Z2Z2, p2->Z); /* Z2Z2 = Z2^2 */\ +\ + mul_##field(a1.X, p1->X, Z2Z2); /* U1 = X1*Z2Z2 */\ + mul_##field(a2.X, p2->X, Z1Z1); /* U2 = X2*Z1Z1 */\ +\ + mul_##field(a1.Y, p1->Y, p2->Z); /* Y1*Z2 */\ + mul_##field(a2.Y, p2->Y, p1->Z); /* Y2*Z1 */\ +\ + mul_##field(a1.Y, a1.Y, Z2Z2); /* S1 = Y1*Z2*Z2Z2 */\ + mul_##field(a2.Y, a2.Y, Z1Z1); /* S2 = Y2*Z1*Z1Z1 */\ +\ + return vec_is_equal(&a1, &a2, sizeof(a1)) & (is_inf1 ^ is_inf2 ^ 1); \ +} + +/* + * https://eprint.iacr.org/2015/1060, algorithm 7 with a twist to handle + * |p3| pointing at either |p1| or |p2|. This is resolved by adding |t5| + * and replacing few first references to |X3| in the formula, up to step + * 21, with it. 12M[+27A], doubling and infinity are handled by the + * formula itself. Infinity is to be encoded as [0, !0, 0]. + */ +#define POINT_PROJ_DADD_IMPL_A0(ptype, bits, field, suffixb) \ +static void ptype##proj_dadd(ptype##proj *p3, const ptype##proj *p1, \ + const ptype##proj *p2) \ +{ \ + vec##bits t0, t1, t2, t3, t4, t5; \ +\ + mul_##field(t0, p1->X, p2->X); /* 1. t0 = X1*X2 */\ + mul_##field(t1, p1->Y, p2->Y); /* 2. t1 = Y1*Y2 */\ + mul_##field(t2, p1->Z, p2->Z); /* 3. t2 = Z1*Z2 */\ + add_##field(t3, p1->X, p1->Y); /* 4. t3 = X1+Y1 */\ + add_##field(t4, p2->X, p2->Y); /* 5. t4 = X2+Y2 */\ + mul_##field(t3, t3, t4); /* 6. t3 = t3*t4 */\ + add_##field(t4, t0, t1); /* 7. t4 = t0+t1 */\ + sub_##field(t3, t3, t4); /* 8. t3 = t3-t4 */\ + add_##field(t4, p1->Y, p1->Z); /* 9. t4 = Y1+Z1 */\ + add_##field(t5, p2->Y, p2->Z); /* 10. t5 = Y2+Z2 */\ + mul_##field(t4, t4, t5); /* 11. t4 = t4*t5 */\ + add_##field(t5, t1, t2); /* 12. t5 = t1+t2 */\ + sub_##field(t4, t4, t5); /* 13. t4 = t4-t5 */\ + add_##field(t5, p1->X, p1->Z); /* 14. t5 = X1+Z1 */\ + add_##field(p3->Y, p2->X, p2->Z); /* 15. Y3 = X2+Z2 */\ + mul_##field(t5, t5, p3->Y); /* 16. t5 = t5*Y3 */\ + add_##field(p3->Y, t0, t2); /* 17. Y3 = t0+t2 */\ + sub_##field(p3->Y, t5, p3->Y); /* 18. Y3 = t5-Y3 */\ + mul_by_3_##field(t0, t0); /* 19-20. t0 = 3*t0 */\ + mul_by_3_##field(t5, t2); /* 21. t5 = 3*t2 */\ + mul_by_b_##suffixb(t2, t5); /* 21. t2 = b*t5 */\ + add_##field(p3->Z, t1, t2); /* 22. Z3 = t1+t2 */\ + sub_##field(t1, t1, t2); /* 23. t1 = t1-t2 */\ + mul_by_3_##field(t5, p3->Y); /* 24. t5 = 3*Y3 */\ + mul_by_b_##suffixb(p3->Y, t5); /* 24. Y3 = b*t5 */\ + mul_##field(p3->X, t4, p3->Y); /* 25. X3 = t4*Y3 */\ + mul_##field(t2, t3, t1); /* 26. t2 = t3*t1 */\ + sub_##field(p3->X, t2, p3->X); /* 27. X3 = t2-X3 */\ + mul_##field(p3->Y, p3->Y, t0); /* 28. Y3 = Y3*t0 */\ + mul_##field(t1, t1, p3->Z); /* 29. t1 = t1*Z3 */\ + add_##field(p3->Y, t1, p3->Y); /* 30. Y3 = t1+Y3 */\ + mul_##field(t0, t0, t3); /* 31. t0 = t0*t3 */\ + mul_##field(p3->Z, p3->Z, t4); /* 32. Z3 = Z3*t4 */\ + add_##field(p3->Z, p3->Z, t0); /* 33. Z3 = Z3+t0 */\ +} + +/* + * https://eprint.iacr.org/2015/1060, algorithm 8 with a twist to handle + * |p2| being infinity encoded as [0, 0]. 11M[+21A]. + */ +#define POINT_PROJ_DADD_AFFINE_IMPL_A0(ptype, bits, field, suffixb) \ +static void ptype##proj_dadd_affine(ptype##proj *out, const ptype##proj *p1, \ + const ptype##_affine *p2) \ +{ \ + ptype##proj p3[1]; \ + vec##bits t0, t1, t2, t3, t4; \ + limb_t p2inf = vec_is_zero(p2, sizeof(*p2)); \ +\ + mul_##field(t0, p1->X, p2->X); /* 1. t0 = X1*X2 */\ + mul_##field(t1, p1->Y, p2->Y); /* 2. t1 = Y1*Y2 */\ + add_##field(t3, p1->X, p1->Y); /* 3. t3 = X1+Y1 */\ + add_##field(t4, p2->X, p2->Y); /* 4. t4 = X2+Y2 */\ + mul_##field(t3, t3, t4); /* 5. t3 = t3*t4 */\ + add_##field(t4, t0, t1); /* 6. t4 = t0+t1 */\ + sub_##field(t3, t3, t4); /* 7. t3 = t3-t4 */\ + mul_##field(t4, p2->Y, p1->Z); /* 8. t4 = Y2*Z1 */\ + add_##field(t4, t4, p1->Y); /* 9. t4 = t4+Y1 */\ + mul_##field(p3->Y, p2->X, p1->Z); /* 10. Y3 = X2*Z1 */\ + add_##field(p3->Y, p3->Y, p1->X); /* 11. Y3 = Y3+X1 */\ + mul_by_3_##field(t0, t0); /* 12-13. t0 = 3*t0 */\ + mul_by_b_##suffixb(t2, p1->Z); /* 14. t2 = b*Z1 */\ + mul_by_3_##field(t2, t2); /* 14. t2 = 3*t2 */\ + add_##field(p3->Z, t1, t2); /* 15. Z3 = t1+t2 */\ + sub_##field(t1, t1, t2); /* 16. t1 = t1-t2 */\ + mul_by_b_##suffixb(t2, p3->Y); /* 17. t2 = b*Y3 */\ + mul_by_3_##field(p3->Y, t2); /* 17. Y3 = 3*t2 */\ + mul_##field(p3->X, t4, p3->Y); /* 18. X3 = t4*Y3 */\ + mul_##field(t2, t3, t1); /* 19. t2 = t3*t1 */\ + sub_##field(p3->X, t2, p3->X); /* 20. X3 = t2-X3 */\ + mul_##field(p3->Y, p3->Y, t0); /* 21. Y3 = Y3*t0 */\ + mul_##field(t1, t1, p3->Z); /* 22. t1 = t1*Z3 */\ + add_##field(p3->Y, t1, p3->Y); /* 23. Y3 = t1+Y3 */\ + mul_##field(t0, t0, t3); /* 24. t0 = t0*t3 */\ + mul_##field(p3->Z, p3->Z, t4); /* 25. Z3 = Z3*t4 */\ + add_##field(p3->Z, p3->Z, t0); /* 26. Z3 = Z3+t0 */\ +\ + vec_select(out, p1, p3, sizeof(*out), p2inf); \ +} + +/* + * https://eprint.iacr.org/2015/1060, algorithm 9 with a twist to handle + * |p3| pointing at |p1|. This is resolved by adding |t3| to hold X*Y + * and reordering operations to bring references to |p1| forward. + * 6M+2S[+13A]. + */ +#define POINT_PROJ_DOUBLE_IMPL_A0(ptype, bits, field, suffixb) \ +static void ptype##proj_double(ptype##proj *p3, const ptype##proj *p1) \ +{ \ + vec##bits t0, t1, t2, t3; \ +\ + sqr_##field(t0, p1->Y); /* 1. t0 = Y*Y */\ + mul_##field(t1, p1->Y, p1->Z); /* 5. t1 = Y*Z */\ + sqr_##field(t2, p1->Z); /* 6. t2 = Z*Z */\ + mul_##field(t3, p1->X, p1->Y); /* 16. t3 = X*Y */\ + lshift_##field(p3->Z, t0, 3); /* 2-4. Z3 = 8*t0 */\ + mul_by_b_##suffixb(p3->X, t2); /* 7. t2 = b*t2 */\ + mul_by_3_##field(t2, p3->X); /* 7. t2 = 3*t2 */\ + mul_##field(p3->X, t2, p3->Z); /* 8. X3 = t2*Z3 */\ + add_##field(p3->Y, t0, t2); /* 9. Y3 = t0+t2 */\ + mul_##field(p3->Z, t1, p3->Z); /* 10. Z3 = t1*Z3 */\ + mul_by_3_##field(t2, t2); /* 11-12. t2 = 3*t2 */\ + sub_##field(t0, t0, t2); /* 13. t0 = t0-t2 */\ + mul_##field(p3->Y, t0, p3->Y); /* 14. Y3 = t0*Y3 */\ + add_##field(p3->Y, p3->X, p3->Y); /* 15. Y3 = X3+Y3 */\ + mul_##field(p3->X, t0, t3); /* 17. X3 = t0*t3 */\ + add_##field(p3->X, p3->X, p3->X); /* 18. X3 = X3+X3 */\ +} + +#define POINT_PROJ_TO_JACOBIAN_IMPL(ptype, bits, field) \ +static void ptype##proj_to_Jacobian(ptype *out, const ptype##proj *in) \ +{ \ + vec##bits ZZ; \ +\ + sqr_##field(ZZ, in->Z); \ + mul_##field(out->X, in->X, in->Z); \ + mul_##field(out->Y, in->Y, ZZ); \ + vec_copy(out->Z, in->Z, sizeof(out->Z)); \ +} + +#define POINT_TO_PROJECTIVE_IMPL(ptype, bits, field, one) \ +static void ptype##_to_projective(ptype##proj *out, const ptype *in) \ +{ \ + vec##bits ZZ; \ + limb_t is_inf = vec_is_zero(in->Z, sizeof(in->Z)); \ +\ + sqr_##field(ZZ, in->Z); \ + mul_##field(out->X, in->X, in->Z); \ + vec_select(out->Y, one, in->Y, sizeof(out->Y), is_inf); \ + mul_##field(out->Z, ZZ, in->Z); \ +} + +/******************* !!!!! NOT CONSTANT TIME !!!!! *******************/ + +/* + * http://hyperelliptic.org/EFD/g1p/auto-shortw-xyzz.html#addition-add-2008-s + * http://hyperelliptic.org/EFD/g1p/auto-shortw-xyzz.html#doubling-dbl-2008-s-1 + * with twist to handle either input at infinity. Addition costs 12M+2S, + * while conditional doubling - 4M+6M+3S. + */ +#define POINTXYZZ_DADD_IMPL(ptype, bits, field) \ +static void ptype##xyzz_dadd(ptype##xyzz *p3, const ptype##xyzz *p1, \ + const ptype##xyzz *p2) \ +{ \ + vec##bits U, S, P, R; \ +\ + if (vec_is_zero(p2->ZZZ, 2*sizeof(p2->ZZZ))) { \ + vec_copy(p3, p1, sizeof(*p3)); \ + return; \ + } else if (vec_is_zero(p1->ZZZ, 2*sizeof(p1->ZZZ))) { \ + vec_copy(p3, p2, sizeof(*p3)); \ + return; \ + } \ +\ + mul_##field(U, p1->X, p2->ZZ); /* U1 = X1*ZZ2 */\ + mul_##field(S, p1->Y, p2->ZZZ); /* S1 = Y1*ZZZ2 */\ + mul_##field(P, p2->X, p1->ZZ); /* U2 = X2*ZZ1 */\ + mul_##field(R, p2->Y, p1->ZZZ); /* S2 = Y2*ZZZ1 */\ + sub_##field(P, P, U); /* P = U2-U1 */\ + sub_##field(R, R, S); /* R = S2-S1 */\ +\ + if (!vec_is_zero(P, sizeof(P))) { /* X1!=X2 */\ + vec##bits PP, PPP, Q; /* add |p1| and |p2| */\ +\ + sqr_##field(PP, P); /* PP = P^2 */\ + mul_##field(PPP, PP, P); /* PPP = P*PP */\ + mul_##field(Q, U, PP); /* Q = U1*PP */\ + sqr_##field(p3->X, R); /* R^2 */\ + add_##field(P, Q, Q); \ + sub_##field(p3->X, p3->X, PPP); /* R^2-PPP */\ + sub_##field(p3->X, p3->X, P); /* X3 = R^2-PPP-2*Q */\ + sub_##field(Q, Q, p3->X); \ + mul_##field(Q, Q, R); /* R*(Q-X3) */\ + mul_##field(p3->Y, S, PPP); /* S1*PPP */\ + sub_##field(p3->Y, Q, p3->Y); /* Y3 = R*(Q-X3)-S1*PPP */\ + mul_##field(p3->ZZ, p1->ZZ, p2->ZZ); /* ZZ1*ZZ2 */\ + mul_##field(p3->ZZZ, p1->ZZZ, p2->ZZZ); /* ZZZ1*ZZZ2 */\ + mul_##field(p3->ZZ, p3->ZZ, PP); /* ZZ3 = ZZ1*ZZ2*PP */\ + mul_##field(p3->ZZZ, p3->ZZZ, PPP); /* ZZZ3 = ZZZ1*ZZZ2*PPP */\ + } else if (vec_is_zero(R, sizeof(R))) { /* X1==X2 && Y1==Y2 */\ + vec##bits V, W, M; /* double |p1| */\ +\ + add_##field(U, p1->Y, p1->Y); /* U = 2*Y1 */\ + sqr_##field(V, U); /* V = U^2 */\ + mul_##field(W, V, U); /* W = U*V */\ + mul_##field(S, p1->X, V); /* S = X1*V */\ + sqr_##field(M, p1->X); \ + mul_by_3_##field(M, M); /* M = 3*X1^2[+a*ZZ1^2] */\ + sqr_##field(p3->X, M); \ + add_##field(U, S, S); /* 2*S */\ + sub_##field(p3->X, p3->X, U); /* X3 = M^2-2*S */\ + mul_##field(p3->Y, W, p1->Y); /* W*Y1 */\ + sub_##field(S, S, p3->X); \ + mul_##field(S, S, M); /* M*(S-X3) */\ + sub_##field(p3->Y, S, p3->Y); /* Y3 = M*(S-X3)-W*Y1 */\ + mul_##field(p3->ZZ, p1->ZZ, V); /* ZZ3 = V*ZZ1 */\ + mul_##field(p3->ZZZ, p1->ZZZ, W); /* ZZ3 = W*ZZZ1 */\ + } else { /* X1==X2 && Y1==-Y2 */\ + vec_zero(p3->ZZZ, 2*sizeof(p3->ZZZ)); /* set |p3| to infinity */\ + } \ +} + +/* + * http://hyperelliptic.org/EFD/g1p/auto-shortw-xyzz.html#addition-madd-2008-s + * http://hyperelliptic.org/EFD/g1p/auto-shortw-xyzz.html#doubling-mdbl-2008-s-1 + * with twists to handle even subtractions and either input at infinity. + * Addition costs 8M+2S, while conditional doubling - 2M+4M+3S. + */ +#define POINTXYZZ_DADD_AFFINE_IMPL(ptype, bits, field, one) \ +static void ptype##xyzz_dadd_affine(ptype##xyzz *p3, const ptype##xyzz *p1, \ + const ptype##_affine *p2, \ + bool_t subtract) \ +{ \ + vec##bits P, R; \ +\ + if (vec_is_zero(p2, sizeof(*p2))) { \ + vec_copy(p3, p1, sizeof(*p3)); \ + return; \ + } else if (vec_is_zero(p1->ZZZ, 2*sizeof(p1->ZZZ))) { \ + vec_copy(p3->X, p2->X, 2*sizeof(p3->X));\ + cneg_##field(p3->ZZZ, one, subtract); \ + vec_copy(p3->ZZ, one, sizeof(p3->ZZ)); \ + return; \ + } \ +\ + mul_##field(P, p2->X, p1->ZZ); /* U2 = X2*ZZ1 */\ + mul_##field(R, p2->Y, p1->ZZZ); /* S2 = Y2*ZZZ1 */\ + cneg_##field(R, R, subtract); \ + sub_##field(P, P, p1->X); /* P = U2-X1 */\ + sub_##field(R, R, p1->Y); /* R = S2-Y1 */\ +\ + if (!vec_is_zero(P, sizeof(P))) { /* X1!=X2 */\ + vec##bits PP, PPP, Q; /* add |p2| to |p1| */\ +\ + sqr_##field(PP, P); /* PP = P^2 */\ + mul_##field(PPP, PP, P); /* PPP = P*PP */\ + mul_##field(Q, p1->X, PP); /* Q = X1*PP */\ + sqr_##field(p3->X, R); /* R^2 */\ + add_##field(P, Q, Q); \ + sub_##field(p3->X, p3->X, PPP); /* R^2-PPP */\ + sub_##field(p3->X, p3->X, P); /* X3 = R^2-PPP-2*Q */\ + sub_##field(Q, Q, p3->X); \ + mul_##field(Q, Q, R); /* R*(Q-X3) */\ + mul_##field(p3->Y, p1->Y, PPP); /* Y1*PPP */\ + sub_##field(p3->Y, Q, p3->Y); /* Y3 = R*(Q-X3)-Y1*PPP */\ + mul_##field(p3->ZZ, p1->ZZ, PP); /* ZZ3 = ZZ1*PP */\ + mul_##field(p3->ZZZ, p1->ZZZ, PPP); /* ZZZ3 = ZZZ1*PPP */\ + } else if (vec_is_zero(R, sizeof(R))) { /* X1==X2 && Y1==Y2 */\ + vec##bits U, S, M; /* double |p2| */\ +\ + add_##field(U, p2->Y, p2->Y); /* U = 2*Y1 */\ + sqr_##field(p3->ZZ, U); /* [ZZ3 =] V = U^2 */\ + mul_##field(p3->ZZZ, p3->ZZ, U); /* [ZZZ3 =] W = U*V */\ + mul_##field(S, p2->X, p3->ZZ); /* S = X1*V */\ + sqr_##field(M, p2->X); \ + mul_by_3_##field(M, M); /* M = 3*X1^2[+a] */\ + sqr_##field(p3->X, M); \ + add_##field(U, S, S); /* 2*S */\ + sub_##field(p3->X, p3->X, U); /* X3 = M^2-2*S */\ + mul_##field(p3->Y, p3->ZZZ, p2->Y); /* W*Y1 */\ + sub_##field(S, S, p3->X); \ + mul_##field(S, S, M); /* M*(S-X3) */\ + sub_##field(p3->Y, S, p3->Y); /* Y3 = M*(S-X3)-W*Y1 */\ + cneg_##field(p3->ZZZ, p3->ZZZ, subtract); \ + } else { /* X1==X2 && Y1==-Y2 */\ + vec_zero(p3->ZZZ, 2*sizeof(p3->ZZZ)); /* set |p3| to infinity */\ + } \ +} + +#define POINTXYZZ_TO_JACOBIAN_IMPL(ptype, bits, field) \ +static void ptype##xyzz_to_Jacobian(ptype *out, const ptype##xyzz *in) \ +{ \ + mul_##field(out->X, in->X, in->ZZ); \ + mul_##field(out->Y, in->Y, in->ZZZ); \ + vec_copy(out->Z, in->ZZ, sizeof(out->Z)); \ +} + +#define POINT_TO_XYZZ_IMPL(ptype, bits, field) \ +static void ptype##_to_xyzz(ptype##xyzz *out, const ptype *in) \ +{ \ + vec_copy(out->X, in->X, 2*sizeof(out->X)); \ + sqr_##field(out->ZZ, in->Z); \ + mul_##field(out->ZZZ, out->ZZ, in->Z); \ +} + +#endif diff --git a/crypto/blst_src/errors.h b/crypto/blst_src/errors.h new file mode 100644 index 00000000000..425daeb486f --- /dev/null +++ b/crypto/blst_src/errors.h @@ -0,0 +1,19 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_ERRORS_H__ +#define __BLS12_381_ASM_ERRORS_H__ + +typedef enum { + BLST_SUCCESS = 0, + BLST_BAD_ENCODING, + BLST_POINT_NOT_ON_CURVE, + BLST_POINT_NOT_IN_GROUP, + BLST_AGGR_TYPE_MISMATCH, + BLST_VERIFY_FAIL, + BLST_PK_IS_INFINITY, +} BLST_ERROR; + +#endif diff --git a/crypto/blst_src/exp.c b/crypto/blst_src/exp.c new file mode 100644 index 00000000000..55c5c5a7875 --- /dev/null +++ b/crypto/blst_src/exp.c @@ -0,0 +1,55 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "vect.h" + +/* + * |out| = |inp|^|pow|, small footprint, public exponent + */ +static void exp_mont_384(vec384 out, const vec384 inp, const byte *pow, + size_t pow_bits, const vec384 p, limb_t n0) +{ +#if 1 + vec384 ret; + + vec_copy(ret, inp, sizeof(ret)); /* ret = inp^1 */ + --pow_bits; /* most significant bit is set, skip over */ + while (pow_bits--) { + sqr_mont_384(ret, ret, p, n0); + if (is_bit_set(pow, pow_bits)) + mul_mont_384(ret, ret, inp, p, n0); + } + vec_copy(out, ret, sizeof(ret)); /* out = ret */ +#else + unsigned int i; + vec384 sqr; + + vec_copy(sqr, inp, sizeof(sqr)); + for (i = 0; !is_bit_set(pow, i++);) + sqr_mont_384(sqr, sqr, sqr, p, n0); + vec_copy(out, sqr, sizeof(sqr)); + for (; i < pow_bits; i++) { + sqr_mont_384(sqr, sqr, sqr, p, n0); + if (is_bit_set(pow, i)) + mul_mont_384(out, out, sqr, p, n0); + } +#endif +} + +static void exp_mont_384x(vec384x out, const vec384x inp, const byte *pow, + size_t pow_bits, const vec384 p, limb_t n0) +{ + vec384x ret; + + vec_copy(ret, inp, sizeof(ret)); /* |ret| = |inp|^1 */ + --pow_bits; /* most significant bit is accounted for, skip over */ + while (pow_bits--) { + sqr_mont_384x(ret, ret, p, n0); + if (is_bit_set(pow, pow_bits)) + mul_mont_384x(ret, ret, inp, p, n0); + } + vec_copy(out, ret, sizeof(ret)); /* |out| = |ret| */ +} diff --git a/crypto/blst_src/exports.c b/crypto/blst_src/exports.c new file mode 100644 index 00000000000..ad720999883 --- /dev/null +++ b/crypto/blst_src/exports.c @@ -0,0 +1,559 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * Why this file? Overall goal is to ensure that all internal calls + * remain internal after linking application. This is to both + * + * a) minimize possibility of external name conflicts (since all + * non-blst-prefixed and [assembly subroutines] remain static); + * b) preclude possibility of unintentional internal reference + * overload in shared library context (one can achieve same + * effect with -Bsymbolic, but we don't want to rely on end-user + * to remember to use it); + */ + +#include "fields.h" +#include "bytes.h" + +/* + * BLS12-381-specifc Fr shortcuts to assembly. + */ +void blst_fr_add(vec256 ret, const vec256 a, const vec256 b) +{ add_mod_256(ret, a, b, BLS12_381_r); } + +void blst_fr_sub(vec256 ret, const vec256 a, const vec256 b) +{ sub_mod_256(ret, a, b, BLS12_381_r); } + +void blst_fr_mul_by_3(vec256 ret, const vec256 a) +{ mul_by_3_mod_256(ret, a, BLS12_381_r); } + +void blst_fr_lshift(vec256 ret, const vec256 a, size_t count) +{ lshift_mod_256(ret, a, count, BLS12_381_r); } + +void blst_fr_rshift(vec256 ret, const vec256 a, size_t count) +{ rshift_mod_256(ret, a, count, BLS12_381_r); } + +void blst_fr_mul(vec256 ret, const vec256 a, const vec256 b) +{ mul_mont_sparse_256(ret, a, b, BLS12_381_r, r0); } + +void blst_fr_sqr(vec256 ret, const vec256 a) +{ sqr_mont_sparse_256(ret, a, BLS12_381_r, r0); } + +void blst_fr_cneg(vec256 ret, const vec256 a, int flag) +{ cneg_mod_256(ret, a, is_zero(flag) ^ 1, BLS12_381_r); } + +void blst_fr_to(vec256 ret, const vec256 a) +{ mul_mont_sparse_256(ret, a, BLS12_381_rRR, BLS12_381_r, r0); } + +void blst_fr_from(vec256 ret, const vec256 a) +{ from_mont_256(ret, a, BLS12_381_r, r0); } + +void blst_fr_from_scalar(vec256 ret, const pow256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if ((uptr_t)ret == (uptr_t)a && is_endian.little) { + mul_mont_sparse_256(ret, (const limb_t *)a, BLS12_381_rRR, + BLS12_381_r, r0); + } else { + vec256 out; + limbs_from_le_bytes(out, a, 32); + mul_mont_sparse_256(ret, out, BLS12_381_rRR, BLS12_381_r, r0); + vec_zero(out, sizeof(out)); + } +} + +void blst_scalar_from_fr(pow256 ret, const vec256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if ((uptr_t)ret == (uptr_t)a && is_endian.little) { + from_mont_256((limb_t *)ret, a, BLS12_381_r, r0); + } else { + vec256 out; + from_mont_256(out, a, BLS12_381_r, r0); + le_bytes_from_limbs(ret, out, 32); + vec_zero(out, sizeof(out)); + } +} + +int blst_scalar_fr_check(const pow256 a) +{ return (int)(check_mod_256(a, BLS12_381_r) | + bytes_are_zero(a, sizeof(pow256))); +} + +int blst_sk_check(const pow256 a) +{ return (int)check_mod_256(a, BLS12_381_r); } + +int blst_sk_add_n_check(pow256 ret, const pow256 a, const pow256 b) +{ return (int)add_n_check_mod_256(ret, a, b, BLS12_381_r); } + +int blst_sk_sub_n_check(pow256 ret, const pow256 a, const pow256 b) +{ return (int)sub_n_check_mod_256(ret, a, b, BLS12_381_r); } + +int blst_sk_mul_n_check(pow256 ret, const pow256 a, const pow256 b) +{ + vec256 a_fr, b_fr; + const union { + long one; + char little; + } is_endian = { 1 }; + + if (((size_t)a|(size_t)b)%sizeof(limb_t) != 0 || !is_endian.little) { + limbs_from_le_bytes(a_fr, a, sizeof(a_fr)); + limbs_from_le_bytes(b_fr, b, sizeof(a_fr)); + a = (const byte *)a_fr; + b = (const byte *)b_fr; + } + mul_mont_sparse_256(a_fr, (const limb_t *)a, BLS12_381_rRR, + BLS12_381_r, r0); + mul_mont_sparse_256(b_fr, (const limb_t *)b, BLS12_381_rRR, + BLS12_381_r, r0); + mul_mont_sparse_256(a_fr, a_fr, b_fr, BLS12_381_r, r0); + from_mont_256(a_fr, a_fr, BLS12_381_r, r0); + le_bytes_from_limbs(ret, a_fr, sizeof(a_fr)); + + return (int)(vec_is_zero(a_fr, sizeof(a_fr)) ^ 1); +} + +void blst_sk_inverse(pow256 ret, const pow256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if (((size_t)a|(size_t)ret)%sizeof(limb_t) == 0 && is_endian.little) { + limb_t *out = (limb_t *)ret; + mul_mont_sparse_256(out, (const limb_t *)a, BLS12_381_rRR, + BLS12_381_r, r0); + reciprocal_fr(out, out); + from_mont_256(out, out, BLS12_381_r, r0); + } else { + vec256 out; + limbs_from_le_bytes(out, a, 32); + mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); + reciprocal_fr(out, out); + from_mont_256(out, out, BLS12_381_r, r0); + le_bytes_from_limbs(ret, out, 32); + vec_zero(out, sizeof(out)); + } +} + +/* + * BLS12-381-specifc Fp shortcuts to assembly. + */ +void blst_fp_add(vec384 ret, const vec384 a, const vec384 b) +{ add_fp(ret, a, b); } + +void blst_fp_sub(vec384 ret, const vec384 a, const vec384 b) +{ sub_fp(ret, a, b); } + +void blst_fp_mul_by_3(vec384 ret, const vec384 a) +{ mul_by_3_fp(ret, a); } + +void blst_fp_mul_by_8(vec384 ret, const vec384 a) +{ mul_by_8_fp(ret, a); } + +void blst_fp_lshift(vec384 ret, const vec384 a, size_t count) +{ lshift_fp(ret, a, count); } + +void blst_fp_mul(vec384 ret, const vec384 a, const vec384 b) +{ mul_fp(ret, a, b); } + +void blst_fp_sqr(vec384 ret, const vec384 a) +{ sqr_fp(ret, a); } + +void blst_fp_cneg(vec384 ret, const vec384 a, int flag) +{ cneg_fp(ret, a, is_zero(flag) ^ 1); } + +void blst_fp_to(vec384 ret, const vec384 a) +{ mul_fp(ret, a, BLS12_381_RR); } + +void blst_fp_from(vec384 ret, const vec384 a) +{ from_fp(ret, a); } + +/* + * Fp serialization/deserialization. + */ +void blst_fp_from_uint32(vec384 ret, const unsigned int a[12]) +{ + if (sizeof(limb_t) == 8) { + int i; + for (i = 0; i < 6; i++) + ret[i] = a[2*i] | ((limb_t)a[2*i+1] << (32 & (8*sizeof(limb_t)-1))); + a = (const unsigned int *)ret; + } + mul_fp(ret, (const limb_t *)a, BLS12_381_RR); +} + +void blst_uint32_from_fp(unsigned int ret[12], const vec384 a) +{ + if (sizeof(limb_t) == 4) { + from_fp((limb_t *)ret, a); + } else { + vec384 out; + int i; + + from_fp(out, a); + for (i = 0; i < 6; i++) { + limb_t limb = out[i]; + ret[2*i] = (unsigned int)limb; + ret[2*i+1] = (unsigned int)(limb >> (32 & (8*sizeof(limb_t)-1))); + } + } +} + +void blst_fp_from_uint64(vec384 ret, const unsigned long long a[6]) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if (sizeof(limb_t) == 4 && !is_endian.little) { + int i; + for (i = 0; i < 6; i++) { + unsigned long long limb = a[i]; + ret[2*i] = (limb_t)limb; + ret[2*i+1] = (limb_t)(limb >> 32); + } + a = (const unsigned long long *)ret; + } + mul_fp(ret, (const limb_t *)a, BLS12_381_RR); +} + +void blst_uint64_from_fp(unsigned long long ret[6], const vec384 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if (sizeof(limb_t) == 8 || is_endian.little) { + from_fp((limb_t *)ret, a); + } else { + vec384 out; + int i; + + from_fp(out, a); + for (i = 0; i < 6; i++) + ret[i] = out[2*i] | ((unsigned long long)out[2*i+1] << 32); + } +} + +void blst_fp_from_bendian(vec384 ret, const unsigned char a[48]) +{ + vec384 out; + + limbs_from_be_bytes(out, a, sizeof(vec384)); + mul_fp(ret, out, BLS12_381_RR); +} + +void blst_bendian_from_fp(unsigned char ret[48], const vec384 a) +{ + vec384 out; + + from_fp(out, a); + be_bytes_from_limbs(ret, out, sizeof(vec384)); +} + +void blst_fp_from_lendian(vec384 ret, const unsigned char a[48]) +{ + vec384 out; + + limbs_from_le_bytes(out, a, sizeof(vec384)); + mul_fp(ret, out, BLS12_381_RR); +} + +void blst_lendian_from_fp(unsigned char ret[48], const vec384 a) +{ + vec384 out; + + from_fp(out, a); + le_bytes_from_limbs(ret, out, sizeof(vec384)); +} + +/* + * BLS12-381-specifc Fp2 shortcuts to assembly. + */ +void blst_fp2_add(vec384x ret, const vec384x a, const vec384x b) +{ add_fp2(ret, a, b); } + +void blst_fp2_sub(vec384x ret, const vec384x a, const vec384x b) +{ sub_fp2(ret, a, b); } + +void blst_fp2_mul_by_3(vec384x ret, const vec384x a) +{ mul_by_3_fp2(ret, a); } + +void blst_fp2_mul_by_8(vec384x ret, const vec384x a) +{ mul_by_8_fp2(ret, a); } + +void blst_fp2_lshift(vec384x ret, const vec384x a, size_t count) +{ lshift_fp2(ret, a, count); } + +void blst_fp2_mul(vec384x ret, const vec384x a, const vec384x b) +{ mul_fp2(ret, a, b); } + +void blst_fp2_sqr(vec384x ret, const vec384x a) +{ sqr_fp2(ret, a); } + +void blst_fp2_cneg(vec384x ret, const vec384x a, int flag) +{ cneg_fp2(ret, a, is_zero(flag) ^ 1); } + +/* + * Scalar serialization/deseriazation + */ +void blst_scalar_from_uint32(pow256 ret, const unsigned int a[8]) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + size_t i; + + if ((uptr_t)ret==(uptr_t)a && is_endian.little) + return; + + for(i = 0; i < 8; i++) { + unsigned int w = a[i]; + *ret++ = (byte)w; + *ret++ = (byte)(w >> 8); + *ret++ = (byte)(w >> 16); + *ret++ = (byte)(w >> 24); + } +} + +void blst_uint32_from_scalar(unsigned int ret[8], const pow256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + size_t i; + + if ((uptr_t)ret==(uptr_t)a && is_endian.little) + return; + + for(i = 0; i < 8; i++) { + unsigned int w = (unsigned int)(*a++); + w |= (unsigned int)(*a++) << 8; + w |= (unsigned int)(*a++) << 16; + w |= (unsigned int)(*a++) << 24; + ret[i] = w; + } +} + +void blst_scalar_from_uint64(pow256 ret, const unsigned long long a[4]) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + size_t i; + + if ((uptr_t)ret==(uptr_t)a && is_endian.little) + return; + + for(i = 0; i < 4; i++) { + unsigned long long w = a[i]; + *ret++ = (byte)w; + *ret++ = (byte)(w >> 8); + *ret++ = (byte)(w >> 16); + *ret++ = (byte)(w >> 24); + *ret++ = (byte)(w >> 32); + *ret++ = (byte)(w >> 40); + *ret++ = (byte)(w >> 48); + *ret++ = (byte)(w >> 56); + } +} + +void blst_uint64_from_scalar(unsigned long long ret[4], const pow256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + size_t i; + + if ((uptr_t)ret==(uptr_t)a && is_endian.little) + return; + + for(i = 0; i < 4; i++) { + unsigned long long w = (unsigned long long)(*a++); + w |= (unsigned long long)(*a++) << 8; + w |= (unsigned long long)(*a++) << 16; + w |= (unsigned long long)(*a++) << 24; + w |= (unsigned long long)(*a++) << 32; + w |= (unsigned long long)(*a++) << 40; + w |= (unsigned long long)(*a++) << 48; + w |= (unsigned long long)(*a++) << 56; + ret[i] = w; + } +} + +void blst_scalar_from_bendian(pow256 ret, const unsigned char a[32]) +{ + vec256 out; + limbs_from_be_bytes(out, a, sizeof(out)); + le_bytes_from_limbs(ret, out, sizeof(out)); + vec_zero(out, sizeof(out)); +} + +void blst_bendian_from_scalar(unsigned char ret[32], const pow256 a) +{ + vec256 out; + limbs_from_le_bytes(out, a, sizeof(out)); + be_bytes_from_limbs(ret, out, sizeof(out)); + vec_zero(out, sizeof(out)); +} + +void blst_scalar_from_lendian(pow256 ret, const unsigned char a[32]) +{ + size_t i; + + if ((uptr_t)ret==(uptr_t)a) + return; + + for (i = 0; i < 32; i++) + ret[i] = a[i]; +} + +void blst_lendian_from_scalar(unsigned char ret[32], const pow256 a) +{ + size_t i; + + if ((uptr_t)ret==(uptr_t)a) + return; + + for (i = 0; i < 32; i++) + ret[i] = a[i]; +} + +void blst_fr_from_uint64(vec256 ret, const unsigned long long a[4]) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if (sizeof(limb_t) == 4 && !is_endian.little) { + int i; + for (i = 0; i < 4; i++) { + unsigned long long limb = a[i]; + ret[2*i] = (limb_t)limb; + ret[2*i+1] = (limb_t)(limb >> 32); + } + a = (const unsigned long long *)ret; + } + mul_mont_sparse_256(ret, (const limb_t *)a, BLS12_381_rRR, BLS12_381_r, r0); +} + +void blst_uint64_from_fr(unsigned long long ret[4], const vec256 a) +{ + const union { + long one; + char little; + } is_endian = { 1 }; + + if (sizeof(limb_t) == 8 || is_endian.little) { + from_mont_256((limb_t *)ret, a, BLS12_381_r, r0); + } else { + vec256 out; + int i; + + from_mont_256(out, a, BLS12_381_r, r0); + for (i = 0; i < 4; i++) + ret[i] = out[2*i] | ((unsigned long long)out[2*i+1] << 32); + vec_zero(out, sizeof(out)); + } +} + +int blst_scalar_from_le_bytes(pow256 out, const unsigned char *bytes, size_t n) +{ + struct { vec256 out, digit, radix; } t; + limb_t ret; + + vec_zero(t.out, sizeof(t.out)); + vec_copy(t.radix, BLS12_381_rRR, sizeof(t.radix)); + + while (n > 32) { + limbs_from_le_bytes(t.digit, bytes, 32); + from_mont_256(t.digit, t.digit, BLS12_381_r, r0); + mul_mont_sparse_256(t.digit, t.digit, t.radix, BLS12_381_r, r0); + add_mod_256(t.out, t.out, t.digit, BLS12_381_r); + mul_mont_sparse_256(t.radix, t.radix, BLS12_381_rRR, BLS12_381_r, r0); + bytes += 32; + n -= 32; + } + + vec_zero(t.digit, sizeof(t.digit)); + limbs_from_le_bytes(t.digit, bytes, n); + from_mont_256(t.digit, t.digit, BLS12_381_r, r0); + mul_mont_sparse_256(t.digit, t.digit, t.radix, BLS12_381_r, r0); + add_mod_256(t.out, t.out, t.digit, BLS12_381_r); + + ret = vec_is_zero(t.out, sizeof(t.out)); + le_bytes_from_limbs(out, t.out, 32); + vec_zero(t.out, 2*sizeof(t.out)); + + return (int)(ret^1); +} + +int blst_scalar_from_be_bytes(pow256 out, const unsigned char *bytes, size_t n) +{ + struct { vec256 out, digit, radix; } t; + limb_t ret; + + vec_zero(t.out, sizeof(t.out)); + vec_copy(t.radix, BLS12_381_rRR, sizeof(t.radix)); + + bytes += n; + while (n > 32) { + limbs_from_be_bytes(t.digit, bytes -= 32, 32); + from_mont_256(t.digit, t.digit, BLS12_381_r, r0); + mul_mont_sparse_256(t.digit, t.digit, t.radix, BLS12_381_r, r0); + add_mod_256(t.out, t.out, t.digit, BLS12_381_r); + mul_mont_sparse_256(t.radix, t.radix, BLS12_381_rRR, BLS12_381_r, r0); + n -= 32; + } + + vec_zero(t.digit, sizeof(t.digit)); + limbs_from_be_bytes(t.digit, bytes -= n, n); + from_mont_256(t.digit, t.digit, BLS12_381_r, r0); + mul_mont_sparse_256(t.digit, t.digit, t.radix, BLS12_381_r, r0); + add_mod_256(t.out, t.out, t.digit, BLS12_381_r); + + ret = vec_is_zero(t.out, sizeof(t.out)); + le_bytes_from_limbs(out, t.out, 32); + vec_zero(t.out, 2*sizeof(t.out)); + + return (int)(ret^1); +} + +/* + * Test facilitator + */ +void blst_scalar_from_hexascii(pow256 ret, const char *hex) +{ bytes_from_hexascii(ret, sizeof(pow256), hex); } + +void blst_fr_from_hexascii(vec256 ret, const char *hex) +{ + limbs_from_hexascii(ret, sizeof(vec256), hex); + mul_mont_sparse_256(ret, ret, BLS12_381_rRR, BLS12_381_r, r0); +} + +void blst_fp_from_hexascii(vec384 ret, const char *hex) +{ + limbs_from_hexascii(ret, sizeof(vec384), hex); + mul_fp(ret, ret, BLS12_381_RR); +} diff --git a/crypto/blst_src/fields.h b/crypto/blst_src/fields.h new file mode 100644 index 00000000000..515219f62dd --- /dev/null +++ b/crypto/blst_src/fields.h @@ -0,0 +1,116 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_FIELDS_H__ +#define __BLS12_381_ASM_FIELDS_H__ + +#include "vect.h" +#include "consts.h" + +/* + * BLS12-381-specifc Fp shortcuts to assembly. + */ +static inline void add_fp(vec384 ret, const vec384 a, const vec384 b) +{ add_mod_384(ret, a, b, BLS12_381_P); } + +static inline void sub_fp(vec384 ret, const vec384 a, const vec384 b) +{ sub_mod_384(ret, a, b, BLS12_381_P); } + +static inline void mul_by_3_fp(vec384 ret, const vec384 a) +{ mul_by_3_mod_384(ret, a, BLS12_381_P); } + +static inline void mul_by_8_fp(vec384 ret, const vec384 a) +{ mul_by_8_mod_384(ret, a, BLS12_381_P); } + +static inline void lshift_fp(vec384 ret, const vec384 a, size_t count) +{ lshift_mod_384(ret, a, count, BLS12_381_P); } + +static inline void rshift_fp(vec384 ret, const vec384 a, size_t count) +{ rshift_mod_384(ret, a, count, BLS12_381_P); } + +static inline void div_by_2_fp(vec384 ret, const vec384 a) +{ div_by_2_mod_384(ret, a, BLS12_381_P); } + +static inline void mul_fp(vec384 ret, const vec384 a, const vec384 b) +{ mul_mont_384(ret, a, b, BLS12_381_P, p0); } + +static inline void sqr_fp(vec384 ret, const vec384 a) +{ sqr_mont_384(ret, a, BLS12_381_P, p0); } + +static inline void cneg_fp(vec384 ret, const vec384 a, bool_t flag) +{ cneg_mod_384(ret, a, flag, BLS12_381_P); } + +static inline void from_fp(vec384 ret, const vec384 a) +{ from_mont_384(ret, a, BLS12_381_P, p0); } + +static inline void redc_fp(vec384 ret, const vec768 a) +{ redc_mont_384(ret, a, BLS12_381_P, p0); } + +/* + * BLS12-381-specifc Fp2 shortcuts to assembly. + */ +static inline void add_fp2(vec384x ret, const vec384x a, const vec384x b) +{ add_mod_384x(ret, a, b, BLS12_381_P); } + +static inline void sub_fp2(vec384x ret, const vec384x a, const vec384x b) +{ sub_mod_384x(ret, a, b, BLS12_381_P); } + +static inline void mul_by_3_fp2(vec384x ret, const vec384x a) +{ mul_by_3_mod_384x(ret, a, BLS12_381_P); } + +static inline void mul_by_8_fp2(vec384x ret, const vec384x a) +{ mul_by_8_mod_384x(ret, a, BLS12_381_P); } + +static inline void lshift_fp2(vec384x ret, const vec384x a, size_t count) +{ + lshift_mod_384(ret[0], a[0], count, BLS12_381_P); + lshift_mod_384(ret[1], a[1], count, BLS12_381_P); +} + +static inline void mul_fp2(vec384x ret, const vec384x a, const vec384x b) +{ mul_mont_384x(ret, a, b, BLS12_381_P, p0); } + +static inline void sqr_fp2(vec384x ret, const vec384x a) +{ sqr_mont_384x(ret, a, BLS12_381_P, p0); } + +static inline void cneg_fp2(vec384x ret, const vec384x a, bool_t flag) +{ + cneg_mod_384(ret[0], a[0], flag, BLS12_381_P); + cneg_mod_384(ret[1], a[1], flag, BLS12_381_P); +} + +#define vec_load_global vec_copy + +static void reciprocal_fp(vec384 out, const vec384 inp); +static void flt_reciprocal_fp(vec384 out, const vec384 inp); +static bool_t recip_sqrt_fp(vec384 out, const vec384 inp); +static bool_t sqrt_fp(vec384 out, const vec384 inp); + +static void reciprocal_fp2(vec384x out, const vec384x inp); +static void flt_reciprocal_fp2(vec384x out, const vec384x inp); +static bool_t recip_sqrt_fp2(vec384x out, const vec384x inp, + const vec384x recip_ZZZ, const vec384x magic_ZZZ); +static bool_t sqrt_fp2(vec384x out, const vec384x inp); +static bool_t sqrt_align_fp2(vec384x out, const vec384x ret, + const vec384x sqrt, const vec384x inp); + +typedef vec384x vec384fp2; +typedef vec384fp2 vec384fp6[3]; +typedef vec384fp6 vec384fp12[2]; + +static void sqr_fp12(vec384fp12 ret, const vec384fp12 a); +static void cyclotomic_sqr_fp12(vec384fp12 ret, const vec384fp12 a); +static void mul_fp12(vec384fp12 ret, const vec384fp12 a, const vec384fp12 b); +static void mul_by_xy00z0_fp12(vec384fp12 ret, const vec384fp12 a, + const vec384fp6 xy00z0); +static void conjugate_fp12(vec384fp12 a); +static void inverse_fp12(vec384fp12 ret, const vec384fp12 a); +/* caveat lector! |n| has to be non-zero and not more than 3! */ +static void frobenius_map_fp12(vec384fp12 ret, const vec384fp12 a, size_t n); + +#define neg_fp(r,a) cneg_fp((r),(a),1) +#define neg_fp2(r,a) cneg_fp2((r),(a),1) + +#endif /* __BLS12_381_ASM_FIELDS_H__ */ diff --git a/crypto/blst_src/fp12_tower.c b/crypto/blst_src/fp12_tower.c new file mode 100644 index 00000000000..ab247a8ebf0 --- /dev/null +++ b/crypto/blst_src/fp12_tower.c @@ -0,0 +1,789 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" + +/* + * Fp2 = Fp[u] / (u^2 + 1) + * Fp6 = Fp2[v] / (v^3 - u - 1) + * Fp12 = Fp6[w] / (w^2 - v) + */ + +static inline void mul_by_u_plus_1_fp2(vec384x ret, const vec384x a) +{ mul_by_1_plus_i_mod_384x(ret, a, BLS12_381_P); } + +#if 1 && !defined(__BLST_NO_ASM__) +#define __FP2x2__ +/* + * Fp2x2 is a "widened" version of Fp2, which allows to consolidate + * reductions from several multiplications. In other words instead of + * "mul_redc-mul_redc-add" we get "mul-mul-add-redc," where latter + * addition is double-width... To be more specific this gives ~7-10% + * faster pairing depending on platform... + */ +typedef vec768 vec768x[2]; + +static inline void add_fp2x2(vec768x ret, const vec768x a, const vec768x b) +{ + add_mod_384x384(ret[0], a[0], b[0], BLS12_381_P); + add_mod_384x384(ret[1], a[1], b[1], BLS12_381_P); +} + +static inline void sub_fp2x2(vec768x ret, const vec768x a, const vec768x b) +{ + sub_mod_384x384(ret[0], a[0], b[0], BLS12_381_P); + sub_mod_384x384(ret[1], a[1], b[1], BLS12_381_P); +} + +static inline void mul_by_u_plus_1_fp2x2(vec768x ret, const vec768x a) +{ + /* caveat lector! |ret| may not be same as |a| */ + sub_mod_384x384(ret[0], a[0], a[1], BLS12_381_P); + add_mod_384x384(ret[1], a[0], a[1], BLS12_381_P); +} + +static inline void redc_fp2x2(vec384x ret, const vec768x a) +{ + redc_mont_384(ret[0], a[0], BLS12_381_P, p0); + redc_mont_384(ret[1], a[1], BLS12_381_P, p0); +} + +static void mul_fp2x2(vec768x ret, const vec384x a, const vec384x b) +{ +#if 1 + mul_382x(ret, a, b, BLS12_381_P); /* +~6% in Miller loop */ +#else + union { vec384 x[2]; vec768 x2; } t; + + add_mod_384(t.x[0], a[0], a[1], BLS12_381_P); + add_mod_384(t.x[1], b[0], b[1], BLS12_381_P); + mul_384(ret[1], t.x[0], t.x[1]); + + mul_384(ret[0], a[0], b[0]); + mul_384(t.x2, a[1], b[1]); + + sub_mod_384x384(ret[1], ret[1], ret[0], BLS12_381_P); + sub_mod_384x384(ret[1], ret[1], t.x2, BLS12_381_P); + + sub_mod_384x384(ret[0], ret[0], t.x2, BLS12_381_P); +#endif +} + +static void sqr_fp2x2(vec768x ret, const vec384x a) +{ +#if 1 + sqr_382x(ret, a, BLS12_381_P); /* +~5% in final exponentiation */ +#else + vec384 t0, t1; + + add_mod_384(t0, a[0], a[1], BLS12_381_P); + sub_mod_384(t1, a[0], a[1], BLS12_381_P); + + mul_384(ret[1], a[0], a[1]); + add_mod_384x384(ret[1], ret[1], ret[1], BLS12_381_P); + + mul_384(ret[0], t0, t1); +#endif +} +#endif /* __FP2x2__ */ + +/* + * Fp6 extension + */ +#if defined(__FP2x2__) /* ~10-13% improvement for mul_fp12 and sqr_fp12 */ +typedef vec768x vec768fp6[3]; + +static inline void sub_fp6x2(vec768fp6 ret, const vec768fp6 a, + const vec768fp6 b) +{ + sub_fp2x2(ret[0], a[0], b[0]); + sub_fp2x2(ret[1], a[1], b[1]); + sub_fp2x2(ret[2], a[2], b[2]); +} + +static void mul_fp6x2(vec768fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + vec768x t0, t1, t2; + vec384x aa, bb; + + mul_fp2x2(t0, a[0], b[0]); + mul_fp2x2(t1, a[1], b[1]); + mul_fp2x2(t2, a[2], b[2]); + + /* ret[0] = ((a1 + a2)*(b1 + b2) - a1*b1 - a2*b2)*(u+1) + a0*b0 + = (a1*b2 + a2*b1)*(u+1) + a0*b0 */ + add_fp2(aa, a[1], a[2]); + add_fp2(bb, b[1], b[2]); + mul_fp2x2(ret[0], aa, bb); + sub_fp2x2(ret[0], ret[0], t1); + sub_fp2x2(ret[0], ret[0], t2); + mul_by_u_plus_1_fp2x2(ret[1], ret[0]); /* borrow ret[1] for a moment */ + add_fp2x2(ret[0], ret[1], t0); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + a2*b2*(u+1) + = a0*b1 + a1*b0 + a2*b2*(u+1) */ + add_fp2(aa, a[0], a[1]); + add_fp2(bb, b[0], b[1]); + mul_fp2x2(ret[1], aa, bb); + sub_fp2x2(ret[1], ret[1], t0); + sub_fp2x2(ret[1], ret[1], t1); + mul_by_u_plus_1_fp2x2(ret[2], t2); /* borrow ret[2] for a moment */ + add_fp2x2(ret[1], ret[1], ret[2]); + + /* ret[2] = (a0 + a2)*(b0 + b2) - a0*b0 - a2*b2 + a1*b1 + = a0*b2 + a2*b0 + a1*b1 */ + add_fp2(aa, a[0], a[2]); + add_fp2(bb, b[0], b[2]); + mul_fp2x2(ret[2], aa, bb); + sub_fp2x2(ret[2], ret[2], t0); + sub_fp2x2(ret[2], ret[2], t2); + add_fp2x2(ret[2], ret[2], t1); +} + +static inline void redc_fp6x2(vec384fp6 ret, const vec768fp6 a) +{ + redc_fp2x2(ret[0], a[0]); + redc_fp2x2(ret[1], a[1]); + redc_fp2x2(ret[2], a[2]); +} + +static void mul_fp6(vec384fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + vec768fp6 r; + + mul_fp6x2(r, a, b); + redc_fp6x2(ret, r); /* narrow to normal width */ +} + +static void sqr_fp6(vec384fp6 ret, const vec384fp6 a) +{ + vec768x s0, m01, m12, s2, rx; + + sqr_fp2x2(s0, a[0]); + + mul_fp2x2(m01, a[0], a[1]); + add_fp2x2(m01, m01, m01); + + mul_fp2x2(m12, a[1], a[2]); + add_fp2x2(m12, m12, m12); + + sqr_fp2x2(s2, a[2]); + + /* ret[2] = (a0 + a1 + a2)^2 - a0^2 - a2^2 - 2*(a0*a1) - 2*(a1*a2) + = a1^2 + 2*(a0*a2) */ + add_fp2(ret[2], a[2], a[1]); + add_fp2(ret[2], ret[2], a[0]); + sqr_fp2x2(rx, ret[2]); + sub_fp2x2(rx, rx, s0); + sub_fp2x2(rx, rx, s2); + sub_fp2x2(rx, rx, m01); + sub_fp2x2(rx, rx, m12); + redc_fp2x2(ret[2], rx); + + /* ret[0] = a0^2 + 2*(a1*a2)*(u+1) */ + mul_by_u_plus_1_fp2x2(rx, m12); + add_fp2x2(rx, rx, s0); + redc_fp2x2(ret[0], rx); + + /* ret[1] = a2^2*(u+1) + 2*(a0*a1) */ + mul_by_u_plus_1_fp2x2(rx, s2); + add_fp2x2(rx, rx, m01); + redc_fp2x2(ret[1], rx); +} +#else +static void mul_fp6(vec384fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + vec384x t0, t1, t2, t3, t4, t5; + + mul_fp2(t0, a[0], b[0]); + mul_fp2(t1, a[1], b[1]); + mul_fp2(t2, a[2], b[2]); + + /* ret[0] = ((a1 + a2)*(b1 + b2) - a1*b1 - a2*b2)*(u+1) + a0*b0 + = (a1*b2 + a2*b1)*(u+1) + a0*b0 */ + add_fp2(t4, a[1], a[2]); + add_fp2(t5, b[1], b[2]); + mul_fp2(t3, t4, t5); + sub_fp2(t3, t3, t1); + sub_fp2(t3, t3, t2); + mul_by_u_plus_1_fp2(t3, t3); + /* add_fp2(ret[0], t3, t0); considering possible aliasing... */ + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + a2*b2*(u+1) + = a0*b1 + a1*b0 + a2*b2*(u+1) */ + add_fp2(t4, a[0], a[1]); + add_fp2(t5, b[0], b[1]); + mul_fp2(ret[1], t4, t5); + sub_fp2(ret[1], ret[1], t0); + sub_fp2(ret[1], ret[1], t1); + mul_by_u_plus_1_fp2(t4, t2); + add_fp2(ret[1], ret[1], t4); + + /* ret[2] = (a0 + a2)*(b0 + b2) - a0*b0 - a2*b2 + a1*b1 + = a0*b2 + a2*b0 + a1*b1 */ + add_fp2(t4, a[0], a[2]); + add_fp2(t5, b[0], b[2]); + mul_fp2(ret[2], t4, t5); + sub_fp2(ret[2], ret[2], t0); + sub_fp2(ret[2], ret[2], t2); + add_fp2(ret[2], ret[2], t1); + + add_fp2(ret[0], t3, t0); /* ... moved from above */ +} + +static void sqr_fp6(vec384fp6 ret, const vec384fp6 a) +{ + vec384x s0, m01, m12, s2; + + sqr_fp2(s0, a[0]); + + mul_fp2(m01, a[0], a[1]); + add_fp2(m01, m01, m01); + + mul_fp2(m12, a[1], a[2]); + add_fp2(m12, m12, m12); + + sqr_fp2(s2, a[2]); + + /* ret[2] = (a0 + a1 + a2)^2 - a0^2 - a2^2 - 2*(a0*a1) - 2*(a1*a2) + = a1^2 + 2*(a0*a2) */ + add_fp2(ret[2], a[2], a[1]); + add_fp2(ret[2], ret[2], a[0]); + sqr_fp2(ret[2], ret[2]); + sub_fp2(ret[2], ret[2], s0); + sub_fp2(ret[2], ret[2], s2); + sub_fp2(ret[2], ret[2], m01); + sub_fp2(ret[2], ret[2], m12); + + /* ret[0] = a0^2 + 2*(a1*a2)*(u+1) */ + mul_by_u_plus_1_fp2(ret[0], m12); + add_fp2(ret[0], ret[0], s0); + + /* ret[1] = a2^2*(u+1) + 2*(a0*a1) */ + mul_by_u_plus_1_fp2(ret[1], s2); + add_fp2(ret[1], ret[1], m01); +} +#endif + +static void add_fp6(vec384fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + add_fp2(ret[0], a[0], b[0]); + add_fp2(ret[1], a[1], b[1]); + add_fp2(ret[2], a[2], b[2]); +} + +static void sub_fp6(vec384fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + sub_fp2(ret[0], a[0], b[0]); + sub_fp2(ret[1], a[1], b[1]); + sub_fp2(ret[2], a[2], b[2]); +} + +static void neg_fp6(vec384fp6 ret, const vec384fp6 a) +{ + neg_fp2(ret[0], a[0]); + neg_fp2(ret[1], a[1]); + neg_fp2(ret[2], a[2]); +} + +#if 0 +#define mul_by_v_fp6 mul_by_v_fp6 +static void mul_by_v_fp6(vec384fp6 ret, const vec384fp6 a) +{ + vec384x t; + + mul_by_u_plus_1_fp2(t, a[2]); + vec_copy(ret[2], a[1], sizeof(a[1])); + vec_copy(ret[1], a[0], sizeof(a[0])); + vec_copy(ret[0], t, sizeof(t)); +} +#endif + +/* + * Fp12 extension + */ +#if defined(__FP2x2__) +static void mul_fp12(vec384fp12 ret, const vec384fp12 a, const vec384fp12 b) +{ + vec768fp6 t0, t1, rx; + vec384fp6 t2; + + mul_fp6x2(t0, a[0], b[0]); + mul_fp6x2(t1, a[1], b[1]); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + = a0*b1 + a1*b0 */ + add_fp6(t2, a[0], a[1]); + add_fp6(ret[1], b[0], b[1]); + mul_fp6x2(rx, ret[1], t2); + sub_fp6x2(rx, rx, t0); + sub_fp6x2(rx, rx, t1); + redc_fp6x2(ret[1], rx); + + /* ret[0] = a0*b0 + a1*b1*v */ + mul_by_u_plus_1_fp2x2(rx[0], t1[2]); + add_fp2x2(rx[0], t0[0], rx[0]); + add_fp2x2(rx[1], t0[1], t1[0]); + add_fp2x2(rx[2], t0[2], t1[1]); + redc_fp6x2(ret[0], rx); +} + +static inline void mul_by_0y0_fp6x2(vec768fp6 ret, const vec384fp6 a, + const vec384fp2 b) +{ + mul_fp2x2(ret[1], a[2], b); /* borrow ret[1] for a moment */ + mul_by_u_plus_1_fp2x2(ret[0], ret[1]); + mul_fp2x2(ret[1], a[0], b); + mul_fp2x2(ret[2], a[1], b); +} + +static void mul_by_xy0_fp6x2(vec768fp6 ret, const vec384fp6 a, + const vec384fp6 b) +{ + vec768x t0, t1; + vec384x aa, bb; + + mul_fp2x2(t0, a[0], b[0]); + mul_fp2x2(t1, a[1], b[1]); + + /* ret[0] = ((a1 + a2)*(b1 + 0) - a1*b1 - a2*0)*(u+1) + a0*b0 + = (a1*0 + a2*b1)*(u+1) + a0*b0 */ + mul_fp2x2(ret[1], a[2], b[1]); /* borrow ret[1] for a moment */ + mul_by_u_plus_1_fp2x2(ret[0], ret[1]); + add_fp2x2(ret[0], ret[0], t0); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + a2*0*(u+1) + = a0*b1 + a1*b0 + a2*0*(u+1) */ + add_fp2(aa, a[0], a[1]); + add_fp2(bb, b[0], b[1]); + mul_fp2x2(ret[1], aa, bb); + sub_fp2x2(ret[1], ret[1], t0); + sub_fp2x2(ret[1], ret[1], t1); + + /* ret[2] = (a0 + a2)*(b0 + 0) - a0*b0 - a2*0 + a1*b1 + = a0*0 + a2*b0 + a1*b1 */ + mul_fp2x2(ret[2], a[2], b[0]); + add_fp2x2(ret[2], ret[2], t1); +} + +static void mul_by_xy00z0_fp12(vec384fp12 ret, const vec384fp12 a, + const vec384fp6 xy00z0) +{ + vec768fp6 t0, t1, rr; + vec384fp6 t2; + + mul_by_xy0_fp6x2(t0, a[0], xy00z0); + mul_by_0y0_fp6x2(t1, a[1], xy00z0[2]); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + = a0*b1 + a1*b0 */ + vec_copy(t2[0], xy00z0[0], sizeof(t2[0])); + add_fp2(t2[1], xy00z0[1], xy00z0[2]); + add_fp6(ret[1], a[0], a[1]); + mul_by_xy0_fp6x2(rr, ret[1], t2); + sub_fp6x2(rr, rr, t0); + sub_fp6x2(rr, rr, t1); + redc_fp6x2(ret[1], rr); + + /* ret[0] = a0*b0 + a1*b1*v */ + mul_by_u_plus_1_fp2x2(rr[0], t1[2]); + add_fp2x2(rr[0], t0[0], rr[0]); + add_fp2x2(rr[1], t0[1], t1[0]); + add_fp2x2(rr[2], t0[2], t1[1]); + redc_fp6x2(ret[0], rr); +} +#else +static void mul_fp12(vec384fp12 ret, const vec384fp12 a, const vec384fp12 b) +{ + vec384fp6 t0, t1, t2; + + mul_fp6(t0, a[0], b[0]); + mul_fp6(t1, a[1], b[1]); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + = a0*b1 + a1*b0 */ + add_fp6(t2, a[0], a[1]); + add_fp6(ret[1], b[0], b[1]); + mul_fp6(ret[1], ret[1], t2); + sub_fp6(ret[1], ret[1], t0); + sub_fp6(ret[1], ret[1], t1); + + /* ret[0] = a0*b0 + a1*b1*v */ +#ifdef mul_by_v_fp6 + mul_by_v_fp6(t1, t1); + add_fp6(ret[0], t0, t1); +#else + mul_by_u_plus_1_fp2(t1[2], t1[2]); + add_fp2(ret[0][0], t0[0], t1[2]); + add_fp2(ret[0][1], t0[1], t1[0]); + add_fp2(ret[0][2], t0[2], t1[1]); +#endif +} + +static inline void mul_by_0y0_fp6(vec384fp6 ret, const vec384fp6 a, + const vec384fp2 b) +{ + vec384x t; + + mul_fp2(t, a[2], b); + mul_fp2(ret[2], a[1], b); + mul_fp2(ret[1], a[0], b); + mul_by_u_plus_1_fp2(ret[0], t); +} + +static void mul_by_xy0_fp6(vec384fp6 ret, const vec384fp6 a, const vec384fp6 b) +{ + vec384x t0, t1, /*t2,*/ t3, t4, t5; + + mul_fp2(t0, a[0], b[0]); + mul_fp2(t1, a[1], b[1]); + + /* ret[0] = ((a1 + a2)*(b1 + 0) - a1*b1 - a2*0)*(u+1) + a0*b0 + = (a1*0 + a2*b1)*(u+1) + a0*b0 */ + mul_fp2(t3, a[2], b[1]); + mul_by_u_plus_1_fp2(t3, t3); + /* add_fp2(ret[0], t3, t0); considering possible aliasing... */ + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + a2*0*(u+1) + = a0*b1 + a1*b0 + a2*0*(u+1) */ + add_fp2(t4, a[0], a[1]); + add_fp2(t5, b[0], b[1]); + mul_fp2(ret[1], t4, t5); + sub_fp2(ret[1], ret[1], t0); + sub_fp2(ret[1], ret[1], t1); + + /* ret[2] = (a0 + a2)*(b0 + 0) - a0*b0 - a2*0 + a1*b1 + = a0*0 + a2*b0 + a1*b1 */ + mul_fp2(ret[2], a[2], b[0]); + add_fp2(ret[2], ret[2], t1); + + add_fp2(ret[0], t3, t0); /* ... moved from above */ +} + +static void mul_by_xy00z0_fp12(vec384fp12 ret, const vec384fp12 a, + const vec384fp6 xy00z0) +{ + vec384fp6 t0, t1, t2; + + mul_by_xy0_fp6(t0, a[0], xy00z0); + mul_by_0y0_fp6(t1, a[1], xy00z0[2]); + + /* ret[1] = (a0 + a1)*(b0 + b1) - a0*b0 - a1*b1 + = a0*b1 + a1*b0 */ + vec_copy(t2[0], xy00z0[0], sizeof(t2[0])); + add_fp2(t2[1], xy00z0[1], xy00z0[2]); + add_fp6(ret[1], a[0], a[1]); + mul_by_xy0_fp6(ret[1], ret[1], t2); + sub_fp6(ret[1], ret[1], t0); + sub_fp6(ret[1], ret[1], t1); + + /* ret[0] = a0*b0 + a1*b1*v */ +#ifdef mul_by_v_fp6 + mul_by_v_fp6(t1, t1); + add_fp6(ret[0], t0, t1); +#else + mul_by_u_plus_1_fp2(t1[2], t1[2]); + add_fp2(ret[0][0], t0[0], t1[2]); + add_fp2(ret[0][1], t0[1], t1[0]); + add_fp2(ret[0][2], t0[2], t1[1]); +#endif +} +#endif + +static void sqr_fp12(vec384fp12 ret, const vec384fp12 a) +{ + vec384fp6 t0, t1; + + add_fp6(t0, a[0], a[1]); +#ifdef mul_by_v_fp6 + mul_by_v_fp6(t1, a[1]); + add_fp6(t1, a[0], t1); +#else + mul_by_u_plus_1_fp2(t1[2], a[1][2]); + add_fp2(t1[0], a[0][0], t1[2]); + add_fp2(t1[1], a[0][1], a[1][0]); + add_fp2(t1[2], a[0][2], a[1][1]); +#endif + mul_fp6(t0, t0, t1); + mul_fp6(t1, a[0], a[1]); + + /* ret[1] = 2*(a0*a1) */ + add_fp6(ret[1], t1, t1); + + /* ret[0] = (a0 + a1)*(a0 + a1*v) - a0*a1 - a0*a1*v + = a0^2 + a1^2*v */ + sub_fp6(ret[0], t0, t1); +#ifdef mul_by_v_fp6 + mul_by_v_fp6(t1, t1); + sub_fp6(ret[0], ret[0], t1); +#else + mul_by_u_plus_1_fp2(t1[2], t1[2]); + sub_fp2(ret[0][0], ret[0][0], t1[2]); + sub_fp2(ret[0][1], ret[0][1], t1[0]); + sub_fp2(ret[0][2], ret[0][2], t1[1]); +#endif +} + +static void conjugate_fp12(vec384fp12 a) +{ neg_fp6(a[1], a[1]); } + +static void inverse_fp6(vec384fp6 ret, const vec384fp6 a) +{ + vec384x c0, c1, c2, t0, t1; + + /* c0 = a0^2 - (a1*a2)*(u+1) */ + sqr_fp2(c0, a[0]); + mul_fp2(t0, a[1], a[2]); + mul_by_u_plus_1_fp2(t0, t0); + sub_fp2(c0, c0, t0); + + /* c1 = a2^2*(u+1) - (a0*a1) */ + sqr_fp2(c1, a[2]); + mul_by_u_plus_1_fp2(c1, c1); + mul_fp2(t0, a[0], a[1]); + sub_fp2(c1, c1, t0); + + /* c2 = a1^2 - a0*a2 */ + sqr_fp2(c2, a[1]); + mul_fp2(t0, a[0], a[2]); + sub_fp2(c2, c2, t0); + + /* (a2*c1 + a1*c2)*(u+1) + a0*c0 */ + mul_fp2(t0, c1, a[2]); + mul_fp2(t1, c2, a[1]); + add_fp2(t0, t0, t1); + mul_by_u_plus_1_fp2(t0, t0); + mul_fp2(t1, c0, a[0]); + add_fp2(t0, t0, t1); + + reciprocal_fp2(t1, t0); + + mul_fp2(ret[0], c0, t1); + mul_fp2(ret[1], c1, t1); + mul_fp2(ret[2], c2, t1); +} + +static void inverse_fp12(vec384fp12 ret, const vec384fp12 a) +{ + vec384fp6 t0, t1; + + sqr_fp6(t0, a[0]); + sqr_fp6(t1, a[1]); +#ifdef mul_by_v_fp6 + mul_by_v_fp6(t1, t1); + sub_fp6(t0, t0, t1); +#else + mul_by_u_plus_1_fp2(t1[2], t1[2]); + sub_fp2(t0[0], t0[0], t1[2]); + sub_fp2(t0[1], t0[1], t1[0]); + sub_fp2(t0[2], t0[2], t1[1]); +#endif + + inverse_fp6(t1, t0); + + mul_fp6(ret[0], a[0], t1); + mul_fp6(ret[1], a[1], t1); + neg_fp6(ret[1], ret[1]); +} + +typedef vec384x vec384fp4[2]; + +#if defined(__FP2x2__) +static void sqr_fp4(vec384fp4 ret, const vec384x a0, const vec384x a1) +{ + vec768x t0, t1, t2; + + sqr_fp2x2(t0, a0); + sqr_fp2x2(t1, a1); + add_fp2(ret[1], a0, a1); + + mul_by_u_plus_1_fp2x2(t2, t1); + add_fp2x2(t2, t2, t0); + redc_fp2x2(ret[0], t2); + + sqr_fp2x2(t2, ret[1]); + sub_fp2x2(t2, t2, t0); + sub_fp2x2(t2, t2, t1); + redc_fp2x2(ret[1], t2); +} +#else +static void sqr_fp4(vec384fp4 ret, const vec384x a0, const vec384x a1) +{ + vec384x t0, t1; + + sqr_fp2(t0, a0); + sqr_fp2(t1, a1); + add_fp2(ret[1], a0, a1); + + mul_by_u_plus_1_fp2(ret[0], t1); + add_fp2(ret[0], ret[0], t0); + + sqr_fp2(ret[1], ret[1]); + sub_fp2(ret[1], ret[1], t0); + sub_fp2(ret[1], ret[1], t1); +} +#endif + +static void cyclotomic_sqr_fp12(vec384fp12 ret, const vec384fp12 a) +{ + vec384fp4 t0, t1, t2; + + sqr_fp4(t0, a[0][0], a[1][1]); + sqr_fp4(t1, a[1][0], a[0][2]); + sqr_fp4(t2, a[0][1], a[1][2]); + + sub_fp2(ret[0][0], t0[0], a[0][0]); + add_fp2(ret[0][0], ret[0][0], ret[0][0]); + add_fp2(ret[0][0], ret[0][0], t0[0]); + + sub_fp2(ret[0][1], t1[0], a[0][1]); + add_fp2(ret[0][1], ret[0][1], ret[0][1]); + add_fp2(ret[0][1], ret[0][1], t1[0]); + + sub_fp2(ret[0][2], t2[0], a[0][2]); + add_fp2(ret[0][2], ret[0][2], ret[0][2]); + add_fp2(ret[0][2], ret[0][2], t2[0]); + + mul_by_u_plus_1_fp2(t2[1], t2[1]); + add_fp2(ret[1][0], t2[1], a[1][0]); + add_fp2(ret[1][0], ret[1][0], ret[1][0]); + add_fp2(ret[1][0], ret[1][0], t2[1]); + + add_fp2(ret[1][1], t0[1], a[1][1]); + add_fp2(ret[1][1], ret[1][1], ret[1][1]); + add_fp2(ret[1][1], ret[1][1], t0[1]); + + add_fp2(ret[1][2], t1[1], a[1][2]); + add_fp2(ret[1][2], ret[1][2], ret[1][2]); + add_fp2(ret[1][2], ret[1][2], t1[1]); +} + +/* + * caveat lector! |n| has to be non-zero and not more than 3! + */ +static inline void frobenius_map_fp2(vec384x ret, const vec384x a, size_t n) +{ + vec_copy(ret[0], a[0], sizeof(ret[0])); + cneg_fp(ret[1], a[1], n & 1); +} + +static void frobenius_map_fp6(vec384fp6 ret, const vec384fp6 a, size_t n) +{ + static const vec384x coeffs1[] = { /* (u + 1)^((P^n - 1) / 3) */ + { { 0 }, + { TO_LIMB_T(0xcd03c9e48671f071), TO_LIMB_T(0x5dab22461fcda5d2), + TO_LIMB_T(0x587042afd3851b95), TO_LIMB_T(0x8eb60ebe01bacb9e), + TO_LIMB_T(0x03f97d6e83d050d2), TO_LIMB_T(0x18f0206554638741) } }, + { { TO_LIMB_T(0x30f1361b798a64e8), TO_LIMB_T(0xf3b8ddab7ece5a2a), + TO_LIMB_T(0x16a8ca3ac61577f7), TO_LIMB_T(0xc26a2ff874fd029b), + TO_LIMB_T(0x3636b76660701c6e), TO_LIMB_T(0x051ba4ab241b6160) } }, + { { 0 }, { ONE_MONT_P } } + }; + static const vec384 coeffs2[] = { /* (u + 1)^((2P^n - 2) / 3) */ + { TO_LIMB_T(0x890dc9e4867545c3), TO_LIMB_T(0x2af322533285a5d5), + TO_LIMB_T(0x50880866309b7e2c), TO_LIMB_T(0xa20d1b8c7e881024), + TO_LIMB_T(0x14e4f04fe2db9068), TO_LIMB_T(0x14e56d3f1564853a) }, + { TO_LIMB_T(0xcd03c9e48671f071), TO_LIMB_T(0x5dab22461fcda5d2), + TO_LIMB_T(0x587042afd3851b95), TO_LIMB_T(0x8eb60ebe01bacb9e), + TO_LIMB_T(0x03f97d6e83d050d2), TO_LIMB_T(0x18f0206554638741) }, + { TO_LIMB_T(0x43f5fffffffcaaae), TO_LIMB_T(0x32b7fff2ed47fffd), + TO_LIMB_T(0x07e83a49a2e99d69), TO_LIMB_T(0xeca8f3318332bb7a), + TO_LIMB_T(0xef148d1ea0f4c069), TO_LIMB_T(0x040ab3263eff0206) } + }; + + frobenius_map_fp2(ret[0], a[0], n); + frobenius_map_fp2(ret[1], a[1], n); + frobenius_map_fp2(ret[2], a[2], n); + --n; /* implied ONE_MONT_P at index 0 */ + mul_fp2(ret[1], ret[1], coeffs1[n]); + mul_fp(ret[2][0], ret[2][0], coeffs2[n]); + mul_fp(ret[2][1], ret[2][1], coeffs2[n]); +} + +static void frobenius_map_fp12(vec384fp12 ret, const vec384fp12 a, size_t n) +{ + static const vec384x coeffs[] = { /* (u + 1)^((P^n - 1) / 6) */ + { { TO_LIMB_T(0x07089552b319d465), TO_LIMB_T(0xc6695f92b50a8313), + TO_LIMB_T(0x97e83cccd117228f), TO_LIMB_T(0xa35baecab2dc29ee), + TO_LIMB_T(0x1ce393ea5daace4d), TO_LIMB_T(0x08f2220fb0fb66eb) }, + { TO_LIMB_T(0xb2f66aad4ce5d646), TO_LIMB_T(0x5842a06bfc497cec), + TO_LIMB_T(0xcf4895d42599d394), TO_LIMB_T(0xc11b9cba40a8e8d0), + TO_LIMB_T(0x2e3813cbe5a0de89), TO_LIMB_T(0x110eefda88847faf) } }, + { { TO_LIMB_T(0xecfb361b798dba3a), TO_LIMB_T(0xc100ddb891865a2c), + TO_LIMB_T(0x0ec08ff1232bda8e), TO_LIMB_T(0xd5c13cc6f1ca4721), + TO_LIMB_T(0x47222a47bf7b5c04), TO_LIMB_T(0x0110f184e51c5f59) } }, + { { TO_LIMB_T(0x3e2f585da55c9ad1), TO_LIMB_T(0x4294213d86c18183), + TO_LIMB_T(0x382844c88b623732), TO_LIMB_T(0x92ad2afd19103e18), + TO_LIMB_T(0x1d794e4fac7cf0b9), TO_LIMB_T(0x0bd592fc7d825ec8) }, + { TO_LIMB_T(0x7bcfa7a25aa30fda), TO_LIMB_T(0xdc17dec12a927e7c), + TO_LIMB_T(0x2f088dd86b4ebef1), TO_LIMB_T(0xd1ca2087da74d4a7), + TO_LIMB_T(0x2da2596696cebc1d), TO_LIMB_T(0x0e2b7eedbbfd87d2) } }, + }; + + frobenius_map_fp6(ret[0], a[0], n); + frobenius_map_fp6(ret[1], a[1], n); + --n; /* implied ONE_MONT_P at index 0 */ + mul_fp2(ret[1][0], ret[1][0], coeffs[n]); + mul_fp2(ret[1][1], ret[1][1], coeffs[n]); + mul_fp2(ret[1][2], ret[1][2], coeffs[n]); +} + + +/* + * BLS12-381-specifc Fp12 shortcuts. + */ +void blst_fp12_sqr(vec384fp12 ret, const vec384fp12 a) +{ sqr_fp12(ret, a); } + +void blst_fp12_cyclotomic_sqr(vec384fp12 ret, const vec384fp12 a) +{ cyclotomic_sqr_fp12(ret, a); } + +void blst_fp12_mul(vec384fp12 ret, const vec384fp12 a, const vec384fp12 b) +{ mul_fp12(ret, a, b); } + +void blst_fp12_mul_by_xy00z0(vec384fp12 ret, const vec384fp12 a, + const vec384fp6 xy00z0) +{ mul_by_xy00z0_fp12(ret, a, xy00z0); } + +void blst_fp12_conjugate(vec384fp12 a) +{ conjugate_fp12(a); } + +void blst_fp12_inverse(vec384fp12 ret, const vec384fp12 a) +{ inverse_fp12(ret, a); } + +/* caveat lector! |n| has to be non-zero and not more than 3! */ +void blst_fp12_frobenius_map(vec384fp12 ret, const vec384fp12 a, size_t n) +{ frobenius_map_fp12(ret, a, n); } + +int blst_fp12_is_equal(const vec384fp12 a, const vec384fp12 b) +{ return (int)vec_is_equal(a, b, sizeof(vec384fp12)); } + +int blst_fp12_is_one(const vec384fp12 a) +{ + return (int)(vec_is_equal(a[0][0], BLS12_381_Rx.p2, sizeof(a[0][0])) & + vec_is_zero(a[0][1], sizeof(vec384fp12) - sizeof(a[0][0]))); +} + +const vec384fp12 *blst_fp12_one(void) +{ return (const vec384fp12 *)BLS12_381_Rx.p12; } + +void blst_bendian_from_fp12(unsigned char ret[48*12], const vec384fp12 a) +{ + size_t i, j; + vec384 out; + + for (i = 0; i < 3; i++) { + for (j = 0; j < 2; j++) { + from_fp(out, a[j][i][0]); + be_bytes_from_limbs(ret, out, sizeof(vec384)); ret += 48; + from_fp(out, a[j][i][1]); + be_bytes_from_limbs(ret, out, sizeof(vec384)); ret += 48; + } + } +} + +size_t blst_fp12_sizeof(void) +{ return sizeof(vec384fp12); } diff --git a/crypto/blst_src/hash_to_field.c b/crypto/blst_src/hash_to_field.c new file mode 100644 index 00000000000..6816ea8b922 --- /dev/null +++ b/crypto/blst_src/hash_to_field.c @@ -0,0 +1,177 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consts.h" +#include "sha256.h" + +static const vec384 BLS12_381_RRRR = { /* RR^2 */ + TO_LIMB_T(0xed48ac6bd94ca1e0), TO_LIMB_T(0x315f831e03a7adf8), + TO_LIMB_T(0x9a53352a615e29dd), TO_LIMB_T(0x34c04e5e921e1761), + TO_LIMB_T(0x2512d43565724728), TO_LIMB_T(0x0aa6346091755d4d) +}; + +#ifdef expand_message_xmd +void expand_message_xmd(unsigned char *bytes, size_t len_in_bytes, + const unsigned char *aug, size_t aug_len, + const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len); +#else +static void sha256_init_Zpad(SHA256_CTX *ctx) +{ + ctx->h[0] = 0xda5698beU; + ctx->h[1] = 0x17b9b469U; + ctx->h[2] = 0x62335799U; + ctx->h[3] = 0x779fbecaU; + ctx->h[4] = 0x8ce5d491U; + ctx->h[5] = 0xc0d26243U; + ctx->h[6] = 0xbafef9eaU; + ctx->h[7] = 0x1837a9d8U; + ctx->N = 64; + vec_zero(ctx->buf, sizeof(ctx->buf)); + ctx->off = 0; +} + +static void vec_xor(void *restrict ret, const void *restrict a, + const void *restrict b, size_t num) +{ + limb_t *rp = (limb_t *)ret; + const limb_t *ap = (const limb_t *)a; + const limb_t *bp = (const limb_t *)b; + size_t i; + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) + rp[i] = ap[i] ^ bp[i]; +} + +static void expand_message_xmd(unsigned char *bytes, size_t len_in_bytes, + const unsigned char *aug, size_t aug_len, + const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len) +{ + union { limb_t align; unsigned char c[32]; } b_0; + union { limb_t align; unsigned char c[33+256+31]; } b_i; + unsigned char *p; + size_t i, b_i_bits, b_i_blocks; + SHA256_CTX ctx; + + /* + * compose template for 'strxor(b_0, b_(i-1)) || I2OSP(i, 1) || DST_prime' + */ + if (DST_len > 255) { + sha256_init(&ctx); + sha256_update(&ctx, "H2C-OVERSIZE-DST-", 17); + sha256_update(&ctx, DST, DST_len); + sha256_final(b_0.c, &ctx); + DST = b_0.c, DST_len = 32; + } + b_i_blocks = ((33 + DST_len + 1 + 9) + 63) & -64; + vec_zero(b_i.c + b_i_blocks - 64, 64); + + p = b_i.c + 33; + for (i = 0; i < DST_len; i++) + p[i] = DST[i]; + p[i++] = (unsigned char)DST_len; + p[i++] = 0x80; + p[i+6] = p[i+5] = p[i+4] = p[i+3] = p[i+2] = p[i+1] = p[i+0] = 0; + b_i_bits = (33 + DST_len + 1) * 8; + p = b_i.c + b_i_blocks; + p[-2] = (unsigned char)(b_i_bits >> 8); + p[-1] = (unsigned char)(b_i_bits); + + sha256_init_Zpad(&ctx); /* Z_pad | */ + sha256_update(&ctx, aug, aug_len); /* | aug | */ + sha256_update(&ctx, msg, msg_len); /* | msg | */ + /* | I2OSP(len_in_bytes, 2) || I2OSP(0, 1) || DST_prime */ + b_i.c[30] = (unsigned char)(len_in_bytes >> 8); + b_i.c[31] = (unsigned char)(len_in_bytes); + b_i.c[32] = 0; + sha256_update(&ctx, b_i.c + 30, 3 + DST_len + 1); + sha256_final(b_0.c, &ctx); + + sha256_init_h(ctx.h); + vec_copy(b_i.c, b_0.c, 32); + ++b_i.c[32]; + sha256_block_data_order(ctx.h, b_i.c, b_i_blocks / 64); + sha256_emit(bytes, ctx.h); + + len_in_bytes += 31; /* ell = ceil(len_in_bytes / b_in_bytes), with */ + len_in_bytes /= 32; /* caller being responsible for accordingly large + * buffer. hash_to_field passes one with length + * divisible by 64, remember? which works... */ + while (--len_in_bytes) { + sha256_init_h(ctx.h); + vec_xor(b_i.c, b_0.c, bytes, 32); + bytes += 32; + ++b_i.c[32]; + sha256_block_data_order(ctx.h, b_i.c, b_i_blocks / 64); + sha256_emit(bytes, ctx.h); + } +} +#endif + +/* + * |nelems| is 'count * m' from spec + */ +static void hash_to_field(vec384 elems[], size_t nelems, + const unsigned char *aug, size_t aug_len, + const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len) +{ + size_t L = sizeof(vec384) + 128/8; /* ceil((ceil(log2(p)) + k) / 8) */ + size_t len_in_bytes = L * nelems; /* divisible by 64, hurray! */ +#if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 \ + || defined(__STDC_NO_VLA__) + limb_t *pseudo_random = alloca(len_in_bytes); +#else + limb_t pseudo_random[len_in_bytes/sizeof(limb_t)]; +#endif + unsigned char *bytes; + vec768 elem; + + aug_len = aug!=NULL ? aug_len : 0; + DST_len = DST!=NULL ? DST_len : 0; + + expand_message_xmd((unsigned char *)pseudo_random, len_in_bytes, + aug, aug_len, msg, msg_len, DST, DST_len); + + vec_zero(elem, sizeof(elem)); + bytes = (unsigned char *)pseudo_random; + while (nelems--) { + limbs_from_be_bytes(elem, bytes, L); + bytes += L; + /* + * L-bytes block % P, output is in Montgomery domain... + */ + redc_mont_384(elems[0], elem, BLS12_381_P, p0); + mul_mont_384(elems[0], elems[0], BLS12_381_RRRR, BLS12_381_P, p0); + elems++; + } +} + +void blst_expand_message_xmd(unsigned char *bytes, size_t len_in_bytes, + const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len) +{ + size_t buf_len = (len_in_bytes+31) & ((size_t)0-32); + unsigned char *buf_ptr = bytes; + + if (buf_len > 255*32) + return; + + if (buf_len != len_in_bytes) + buf_ptr = alloca(buf_len); + + expand_message_xmd(buf_ptr, len_in_bytes, NULL, 0, msg, msg_len, + DST, DST_len); + if (buf_ptr != bytes) { + unsigned char *ptr = buf_ptr; + while (len_in_bytes--) + *bytes++ = *ptr++; + vec_zero(buf_ptr, buf_len); + } +} diff --git a/crypto/blst_src/keygen.c b/crypto/blst_src/keygen.c new file mode 100644 index 00000000000..9b62f16b534 --- /dev/null +++ b/crypto/blst_src/keygen.c @@ -0,0 +1,319 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "consts.h" +#include "bytes.h" +#include "sha256.h" + +typedef struct { + SHA256_CTX ctx; + unsigned int h_ipad[8]; + unsigned int h_opad[8]; + union { limb_t l[64/sizeof(limb_t)]; unsigned char c[64]; } tail; +} HMAC_SHA256_CTX; + +static void HMAC_init(HMAC_SHA256_CTX *ctx, const void *K, size_t K_len) +{ + size_t i; + + if (K == NULL) { /* reuse h_ipad and h_opad */ + sha256_hcopy(ctx->ctx.h, ctx->h_ipad); + ctx->ctx.N = 64; + vec_zero(ctx->ctx.buf, sizeof(ctx->ctx.buf)); + ctx->ctx.off = 0; + + return; + } + + vec_zero(ctx->tail.c, sizeof(ctx->tail)); + if (K_len > 64) { + sha256_init(&ctx->ctx); + sha256_update(&ctx->ctx, K, K_len); + sha256_final(ctx->tail.c, &ctx->ctx); + } else { + sha256_bcopy(ctx->tail.c, K, K_len); + } + + for (i = 0; i < 64/sizeof(limb_t); i++) + ctx->tail.l[i] ^= (limb_t)0x3636363636363636; + + sha256_init(&ctx->ctx); + sha256_update(&ctx->ctx, ctx->tail.c, 64); + sha256_hcopy(ctx->h_ipad, ctx->ctx.h); + + for (i = 0; i < 64/sizeof(limb_t); i++) + ctx->tail.l[i] ^= (limb_t)(0x3636363636363636 ^ 0x5c5c5c5c5c5c5c5c); + + sha256_init_h(ctx->h_opad); + sha256_block_data_order(ctx->h_opad, ctx->tail.c, 1); + + vec_zero(ctx->tail.c, sizeof(ctx->tail)); + ctx->tail.c[32] = 0x80; + ctx->tail.c[62] = 3; /* (64+32)*8 in big endian */ + ctx->tail.c[63] = 0; +} + +static void HMAC_update(HMAC_SHA256_CTX *ctx, const unsigned char *inp, + size_t len) +{ sha256_update(&ctx->ctx, inp, len); } + +static void HMAC_final(unsigned char md[32], HMAC_SHA256_CTX *ctx) +{ + sha256_final(ctx->tail.c, &ctx->ctx); + sha256_hcopy(ctx->ctx.h, ctx->h_opad); + sha256_block_data_order(ctx->ctx.h, ctx->tail.c, 1); + sha256_emit(md, ctx->ctx.h); +} + +static void HKDF_Extract(unsigned char PRK[32], + const void *salt, size_t salt_len, + const void *IKM, size_t IKM_len, +#ifndef __BLST_HKDF_TESTMODE__ + int IKM_fixup, +#endif + HMAC_SHA256_CTX *ctx) +{ + unsigned char zero[1] = { 0 }; + + HMAC_init(ctx, salt != NULL ? salt : zero, salt_len); + HMAC_update(ctx, IKM, IKM_len); +#ifndef __BLST_HKDF_TESTMODE__ + if (IKM_fixup) { + /* Section 2.3 KeyGen in BLS-signature draft */ + HMAC_update(ctx, zero, 1); + } +#endif + HMAC_final(PRK, ctx); +} + +static void HKDF_Expand(unsigned char *OKM, size_t L, + const unsigned char PRK[32], + const void *info, size_t info_len, +#ifndef __BLST_HKDF_TESTMODE__ + int info_fixup, +#endif + HMAC_SHA256_CTX *ctx) +{ +#if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 \ + || defined(__STDC_NO_VLA__) + unsigned char *info_prime = alloca(info_len + 2 + 1); +#else + unsigned char info_prime[info_len + 2 + 1]; +#endif + + HMAC_init(ctx, PRK, 32); + + if (info_len != 0) + sha256_bcopy(info_prime, info, info_len); +#ifndef __BLST_HKDF_TESTMODE__ + if (info_fixup) { + /* Section 2.3 KeyGen in BLS-signature draft */ + info_prime[info_len + 0] = (unsigned char)(L >> 8); + info_prime[info_len + 1] = (unsigned char)(L); + info_len += 2; + } +#endif + info_prime[info_len] = 1; /* counter */ + HMAC_update(ctx, info_prime, info_len + 1); + HMAC_final(ctx->tail.c, ctx); + while (L > 32) { + sha256_hcopy((unsigned int *)OKM, (const unsigned int *)ctx->tail.c); + OKM += 32; L -= 32; + ++info_prime[info_len]; /* counter */ + HMAC_init(ctx, NULL, 0); + HMAC_update(ctx, ctx->tail.c, 32); + HMAC_update(ctx, info_prime, info_len + 1); + HMAC_final(ctx->tail.c, ctx); + } + sha256_bcopy(OKM, ctx->tail.c, L); +} + +#ifndef __BLST_HKDF_TESTMODE__ +static void keygen(pow256 SK, const void *IKM, size_t IKM_len, + const void *salt, size_t salt_len, + const void *info, size_t info_len, + int version) +{ + struct { + HMAC_SHA256_CTX ctx; + unsigned char PRK[32], OKM[48]; + vec512 key; + } scratch; + unsigned char salt_prime[32] = "BLS-SIG-KEYGEN-SALT-"; + + if (IKM_len < 32 || (version > 4 && salt == NULL)) { + vec_zero(SK, sizeof(pow256)); + return; + } + + /* + * Vet |info| since some callers were caught to be sloppy, e.g. + * SWIG-4.0-generated Python wrapper... + */ + info_len = info==NULL ? 0 : info_len; + + if (salt == NULL) { + salt = salt_prime; + salt_len = 20; + } + + if (version == 4) { + /* salt = H(salt) */ + sha256_init(&scratch.ctx.ctx); + sha256_update(&scratch.ctx.ctx, salt, salt_len); + sha256_final(salt_prime, &scratch.ctx.ctx); + salt = salt_prime; + salt_len = sizeof(salt_prime); + } + + while (1) { + /* PRK = HKDF-Extract(salt, IKM || I2OSP(0, 1)) */ + HKDF_Extract(scratch.PRK, salt, salt_len, + IKM, IKM_len, 1, &scratch.ctx); + + /* OKM = HKDF-Expand(PRK, key_info || I2OSP(L, 2), L) */ + HKDF_Expand(scratch.OKM, sizeof(scratch.OKM), scratch.PRK, + info, info_len, 1, &scratch.ctx); + + /* SK = OS2IP(OKM) mod r */ + vec_zero(scratch.key, sizeof(scratch.key)); + limbs_from_be_bytes(scratch.key, scratch.OKM, sizeof(scratch.OKM)); + redc_mont_256(scratch.key, scratch.key, BLS12_381_r, r0); + /* + * Given that mul_mont_sparse_256 has special boundary conditions + * it's appropriate to mention that redc_mont_256 output is fully + * reduced at this point. Because we started with 384-bit input, + * one with most significant half smaller than the modulus. + */ + mul_mont_sparse_256(scratch.key, scratch.key, BLS12_381_rRR, + BLS12_381_r, r0); + + if (version < 4 || !vec_is_zero(scratch.key, sizeof(vec256))) + break; + + /* salt = H(salt) */ + sha256_init(&scratch.ctx.ctx); + sha256_update(&scratch.ctx.ctx, salt, salt_len); + sha256_final(salt_prime, &scratch.ctx.ctx); + salt = salt_prime; + salt_len = sizeof(salt_prime); + } + + le_bytes_from_limbs(SK, scratch.key, sizeof(pow256)); + + /* + * scrub the stack just in case next callee inadvertently flashes + * a fragment across application boundary... + */ + vec_zero(&scratch, sizeof(scratch)); +} + +void blst_keygen(pow256 SK, const void *IKM, size_t IKM_len, + const void *info, size_t info_len) +{ keygen(SK, IKM, IKM_len, NULL, 0, info, info_len, 4); } + +void blst_keygen_v3(pow256 SK, const void *IKM, size_t IKM_len, + const void *info, size_t info_len) +{ keygen(SK, IKM, IKM_len, NULL, 0, info, info_len, 3); } + +void blst_keygen_v4_5(pow256 SK, const void *IKM, size_t IKM_len, + const void *salt, size_t salt_len, + const void *info, size_t info_len) +{ keygen(SK, IKM, IKM_len, salt, salt_len, info, info_len, 4); } + +void blst_keygen_v5(pow256 SK, const void *IKM, size_t IKM_len, + const void *salt, size_t salt_len, + const void *info, size_t info_len) +{ keygen(SK, IKM, IKM_len, salt, salt_len, info, info_len, 5); } + +/* + * https://eips.ethereum.org/EIPS/eip-2333 + */ +void blst_derive_master_eip2333(pow256 SK, const void *seed, size_t seed_len) +{ keygen(SK, seed, seed_len, NULL, 0, NULL, 0, 4); } + +static void parent_SK_to_lamport_PK(pow256 PK, const pow256 parent_SK, + unsigned int index) +{ + size_t i; + struct { + HMAC_SHA256_CTX ctx; + SHA256_CTX ret; + unsigned char PRK[32], IKM[32]; + unsigned char lamport[255][32]; + } scratch; + + /* salt = I2OSP(index, 4) */ + unsigned char salt[4] = { (unsigned char)(index>>24), + (unsigned char)(index>>16), + (unsigned char)(index>>8), + (unsigned char)(index) }; + + /* IKM = I2OSP(parent_SK, 32) */ + for (i = 0; i < 32; i++) + scratch.IKM[i] = parent_SK[31-i]; + + /* lamport_0 = IKM_to_lamport_SK(IKM, salt) */ + HKDF_Extract(scratch.PRK, salt, sizeof(salt), scratch.IKM, 32, 0, + &scratch.ctx); + HKDF_Expand(scratch.lamport[0], sizeof(scratch.lamport), + scratch.PRK, NULL, 0, 0, &scratch.ctx); + + vec_zero(scratch.ctx.ctx.buf, sizeof(scratch.ctx.ctx.buf)); + scratch.ctx.ctx.buf[32] = 0x80; + scratch.ctx.ctx.buf[62] = 1; /* 32*8 in big endian */ + scratch.ctx.ctx.buf[63] = 0; + for (i = 0; i < 255; i++) { + /* lamport_PK = lamport_PK | SHA256(lamport_0[i]) */ + sha256_init_h(scratch.ctx.ctx.h); + sha256_bcopy(scratch.ctx.ctx.buf, scratch.lamport[i], 32); + sha256_block_data_order(scratch.ctx.ctx.h, scratch.ctx.ctx.buf, 1); + sha256_emit(scratch.lamport[i], scratch.ctx.ctx.h); + } + + /* compressed_lamport_PK = SHA256(lamport_PK) */ + sha256_init(&scratch.ret); + sha256_update(&scratch.ret, scratch.lamport, sizeof(scratch.lamport)); + + /* not_IKM = flip_bits(IKM) */ + for (i = 0; i< 32; i++) + scratch.IKM[i] = ~scratch.IKM[i]; + + /* lamport_1 = IKM_to_lamport_SK(not_IKM, salt) */ + HKDF_Extract(scratch.PRK, salt, sizeof(salt), scratch.IKM, 32, 0, + &scratch.ctx); + HKDF_Expand(scratch.lamport[0], sizeof(scratch.lamport), + scratch.PRK, NULL, 0, 0, &scratch.ctx); + + vec_zero(scratch.ctx.ctx.buf, sizeof(scratch.ctx.ctx.buf)); + scratch.ctx.ctx.buf[32] = 0x80; + scratch.ctx.ctx.buf[62] = 1; + for (i = 0; i < 255; i++) { + /* lamport_PK = lamport_PK | SHA256(lamport_1[i]) */ + sha256_init_h(scratch.ctx.ctx.h); + sha256_bcopy(scratch.ctx.ctx.buf, scratch.lamport[i], 32); + sha256_block_data_order(scratch.ctx.ctx.h, scratch.ctx.ctx.buf, 1); + sha256_emit(scratch.lamport[i], scratch.ctx.ctx.h); + } + + /* compressed_lamport_PK = SHA256(lamport_PK) */ + sha256_update(&scratch.ret, scratch.lamport, sizeof(scratch.lamport)); + sha256_final(PK, &scratch.ret); + + /* + * scrub the stack just in case next callee inadvertently flashes + * a fragment across application boundary... + */ + vec_zero(&scratch, sizeof(scratch)); +} + +void blst_derive_child_eip2333(pow256 SK, const pow256 parent_SK, + unsigned int child_index) +{ + parent_SK_to_lamport_PK(SK, parent_SK, child_index); + keygen(SK, SK, sizeof(pow256), NULL, 0, NULL, 0, 4); +} +#endif diff --git a/crypto/blst_src/map_to_g1.c b/crypto/blst_src/map_to_g1.c new file mode 100644 index 00000000000..6613d68bb29 --- /dev/null +++ b/crypto/blst_src/map_to_g1.c @@ -0,0 +1,559 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "point.h" +#include "fields.h" + +/* + * y^2 = x^3 + A'*x + B', isogenous one + */ +static const vec384 Aprime_E1 = { + /* (0x00144698a3b8e9433d693a02c96d4982b0ea985383ee66a8 + d8e8981aefd881ac98936f8da0e0f97f5cf428082d584c1d << 384) % P */ + TO_LIMB_T(0x2f65aa0e9af5aa51), TO_LIMB_T(0x86464c2d1e8416c3), + TO_LIMB_T(0xb85ce591b7bd31e2), TO_LIMB_T(0x27e11c91b5f24e7c), + TO_LIMB_T(0x28376eda6bfc1835), TO_LIMB_T(0x155455c3e5071d85) +}; +static const vec384 Bprime_E1 = { + /* (0x12e2908d11688030018b12e8753eee3b2016c1f0f24f4070 + a0b9c14fcef35ef55a23215a316ceaa5d1cc48e98e172be0 << 384) % P */ + TO_LIMB_T(0xfb996971fe22a1e0), TO_LIMB_T(0x9aa93eb35b742d6f), + TO_LIMB_T(0x8c476013de99c5c4), TO_LIMB_T(0x873e27c3a221e571), + TO_LIMB_T(0xca72b5e45a52d888), TO_LIMB_T(0x06824061418a386b) +}; + +static void map_fp_times_Zz(vec384 map[], const vec384 isogeny_map[], + const vec384 Zz_powers[], size_t n) +{ + while (n--) + mul_fp(map[n], isogeny_map[n], Zz_powers[n]); +} + +static void map_fp(vec384 acc, const vec384 x, const vec384 map[], size_t n) +{ + while (n--) { + mul_fp(acc, acc, x); + add_fp(acc, acc, map[n]); + } +} + +static void isogeny_map_to_E1(POINTonE1 *out, const POINTonE1 *p) +{ + /* + * x = x_num / x_den, where + * x_num = k_(1,11) * x'^11 + k_(1,10) * x'^10 + k_(1,9) * x'^9 + + * ... + k_(1,0) + * ... + */ + static const vec384 isogeny_map_x_num[] = { /* (k_(1,*)<<384) % P */ + { TO_LIMB_T(0x4d18b6f3af00131c), TO_LIMB_T(0x19fa219793fee28c), + TO_LIMB_T(0x3f2885f1467f19ae), TO_LIMB_T(0x23dcea34f2ffb304), + TO_LIMB_T(0xd15b58d2ffc00054), TO_LIMB_T(0x0913be200a20bef4) }, + { TO_LIMB_T(0x898985385cdbbd8b), TO_LIMB_T(0x3c79e43cc7d966aa), + TO_LIMB_T(0x1597e193f4cd233a), TO_LIMB_T(0x8637ef1e4d6623ad), + TO_LIMB_T(0x11b22deed20d827b), TO_LIMB_T(0x07097bc5998784ad) }, + { TO_LIMB_T(0xa542583a480b664b), TO_LIMB_T(0xfc7169c026e568c6), + TO_LIMB_T(0x5ba2ef314ed8b5a6), TO_LIMB_T(0x5b5491c05102f0e7), + TO_LIMB_T(0xdf6e99707d2a0079), TO_LIMB_T(0x0784151ed7605524) }, + { TO_LIMB_T(0x494e212870f72741), TO_LIMB_T(0xab9be52fbda43021), + TO_LIMB_T(0x26f5577994e34c3d), TO_LIMB_T(0x049dfee82aefbd60), + TO_LIMB_T(0x65dadd7828505289), TO_LIMB_T(0x0e93d431ea011aeb) }, + { TO_LIMB_T(0x90ee774bd6a74d45), TO_LIMB_T(0x7ada1c8a41bfb185), + TO_LIMB_T(0x0f1a8953b325f464), TO_LIMB_T(0x104c24211be4805c), + TO_LIMB_T(0x169139d319ea7a8f), TO_LIMB_T(0x09f20ead8e532bf6) }, + { TO_LIMB_T(0x6ddd93e2f43626b7), TO_LIMB_T(0xa5482c9aa1ccd7bd), + TO_LIMB_T(0x143245631883f4bd), TO_LIMB_T(0x2e0a94ccf77ec0db), + TO_LIMB_T(0xb0282d480e56489f), TO_LIMB_T(0x18f4bfcbb4368929) }, + { TO_LIMB_T(0x23c5f0c953402dfd), TO_LIMB_T(0x7a43ff6958ce4fe9), + TO_LIMB_T(0x2c390d3d2da5df63), TO_LIMB_T(0xd0df5c98e1f9d70f), + TO_LIMB_T(0xffd89869a572b297), TO_LIMB_T(0x1277ffc72f25e8fe) }, + { TO_LIMB_T(0x79f4f0490f06a8a6), TO_LIMB_T(0x85f894a88030fd81), + TO_LIMB_T(0x12da3054b18b6410), TO_LIMB_T(0xe2a57f6505880d65), + TO_LIMB_T(0xbba074f260e400f1), TO_LIMB_T(0x08b76279f621d028) }, + { TO_LIMB_T(0xe67245ba78d5b00b), TO_LIMB_T(0x8456ba9a1f186475), + TO_LIMB_T(0x7888bff6e6b33bb4), TO_LIMB_T(0xe21585b9a30f86cb), + TO_LIMB_T(0x05a69cdcef55feee), TO_LIMB_T(0x09e699dd9adfa5ac) }, + { TO_LIMB_T(0x0de5c357bff57107), TO_LIMB_T(0x0a0db4ae6b1a10b2), + TO_LIMB_T(0xe256bb67b3b3cd8d), TO_LIMB_T(0x8ad456574e9db24f), + TO_LIMB_T(0x0443915f50fd4179), TO_LIMB_T(0x098c4bf7de8b6375) }, + { TO_LIMB_T(0xe6b0617e7dd929c7), TO_LIMB_T(0xfe6e37d442537375), + TO_LIMB_T(0x1dafdeda137a489e), TO_LIMB_T(0xe4efd1ad3f767ceb), + TO_LIMB_T(0x4a51d8667f0fe1cf), TO_LIMB_T(0x054fdf4bbf1d821c) }, + { TO_LIMB_T(0x72db2a50658d767b), TO_LIMB_T(0x8abf91faa257b3d5), + TO_LIMB_T(0xe969d6833764ab47), TO_LIMB_T(0x464170142a1009eb), + TO_LIMB_T(0xb14f01aadb30be2f), TO_LIMB_T(0x18ae6a856f40715d) } + }; + /* ... + * x_den = x'^10 + k_(2,9) * x'^9 + k_(2,8) * x'^8 + ... + k_(2,0) + */ + static const vec384 isogeny_map_x_den[] = { /* (k_(2,*)<<384) % P */ + { TO_LIMB_T(0xb962a077fdb0f945), TO_LIMB_T(0xa6a9740fefda13a0), + TO_LIMB_T(0xc14d568c3ed6c544), TO_LIMB_T(0xb43fc37b908b133e), + TO_LIMB_T(0x9c0b3ac929599016), TO_LIMB_T(0x0165aa6c93ad115f) }, + { TO_LIMB_T(0x23279a3ba506c1d9), TO_LIMB_T(0x92cfca0a9465176a), + TO_LIMB_T(0x3b294ab13755f0ff), TO_LIMB_T(0x116dda1c5070ae93), + TO_LIMB_T(0xed4530924cec2045), TO_LIMB_T(0x083383d6ed81f1ce) }, + { TO_LIMB_T(0x9885c2a6449fecfc), TO_LIMB_T(0x4a2b54ccd37733f0), + TO_LIMB_T(0x17da9ffd8738c142), TO_LIMB_T(0xa0fba72732b3fafd), + TO_LIMB_T(0xff364f36e54b6812), TO_LIMB_T(0x0f29c13c660523e2) }, + { TO_LIMB_T(0xe349cc118278f041), TO_LIMB_T(0xd487228f2f3204fb), + TO_LIMB_T(0xc9d325849ade5150), TO_LIMB_T(0x43a92bd69c15c2df), + TO_LIMB_T(0x1c2c7844bc417be4), TO_LIMB_T(0x12025184f407440c) }, + { TO_LIMB_T(0x587f65ae6acb057b), TO_LIMB_T(0x1444ef325140201f), + TO_LIMB_T(0xfbf995e71270da49), TO_LIMB_T(0xccda066072436a42), + TO_LIMB_T(0x7408904f0f186bb2), TO_LIMB_T(0x13b93c63edf6c015) }, + { TO_LIMB_T(0xfb918622cd141920), TO_LIMB_T(0x4a4c64423ecaddb4), + TO_LIMB_T(0x0beb232927f7fb26), TO_LIMB_T(0x30f94df6f83a3dc2), + TO_LIMB_T(0xaeedd424d780f388), TO_LIMB_T(0x06cc402dd594bbeb) }, + { TO_LIMB_T(0xd41f761151b23f8f), TO_LIMB_T(0x32a92465435719b3), + TO_LIMB_T(0x64f436e888c62cb9), TO_LIMB_T(0xdf70a9a1f757c6e4), + TO_LIMB_T(0x6933a38d5b594c81), TO_LIMB_T(0x0c6f7f7237b46606) }, + { TO_LIMB_T(0x693c08747876c8f7), TO_LIMB_T(0x22c9850bf9cf80f0), + TO_LIMB_T(0x8e9071dab950c124), TO_LIMB_T(0x89bc62d61c7baf23), + TO_LIMB_T(0xbc6be2d8dad57c23), TO_LIMB_T(0x17916987aa14a122) }, + { TO_LIMB_T(0x1be3ff439c1316fd), TO_LIMB_T(0x9965243a7571dfa7), + TO_LIMB_T(0xc7f7f62962f5cd81), TO_LIMB_T(0x32c6aa9af394361c), + TO_LIMB_T(0xbbc2ee18e1c227f4), TO_LIMB_T(0x0c102cbac531bb34) }, + { TO_LIMB_T(0x997614c97bacbf07), TO_LIMB_T(0x61f86372b99192c0), + TO_LIMB_T(0x5b8c95fc14353fc3), TO_LIMB_T(0xca2b066c2a87492f), + TO_LIMB_T(0x16178f5bbf698711), TO_LIMB_T(0x12a6dcd7f0f4e0e8) } + }; + /* + * y = y' * y_num / y_den, where + * y_num = k_(3,15) * x'^15 + k_(3,14) * x'^14 + k_(3,13) * x'^13 + + * ... + k_(3,0) + * ... + */ + static const vec384 isogeny_map_y_num[] = { /* (k_(3,*)<<384) % P */ + { TO_LIMB_T(0x2b567ff3e2837267), TO_LIMB_T(0x1d4d9e57b958a767), + TO_LIMB_T(0xce028fea04bd7373), TO_LIMB_T(0xcc31a30a0b6cd3df), + TO_LIMB_T(0x7d7b18a682692693), TO_LIMB_T(0x0d300744d42a0310) }, + { TO_LIMB_T(0x99c2555fa542493f), TO_LIMB_T(0xfe7f53cc4874f878), + TO_LIMB_T(0x5df0608b8f97608a), TO_LIMB_T(0x14e03832052b49c8), + TO_LIMB_T(0x706326a6957dd5a4), TO_LIMB_T(0x0a8dadd9c2414555) }, + { TO_LIMB_T(0x13d942922a5cf63a), TO_LIMB_T(0x357e33e36e261e7d), + TO_LIMB_T(0xcf05a27c8456088d), TO_LIMB_T(0x0000bd1de7ba50f0), + TO_LIMB_T(0x83d0c7532f8c1fde), TO_LIMB_T(0x13f70bf38bbf2905) }, + { TO_LIMB_T(0x5c57fd95bfafbdbb), TO_LIMB_T(0x28a359a65e541707), + TO_LIMB_T(0x3983ceb4f6360b6d), TO_LIMB_T(0xafe19ff6f97e6d53), + TO_LIMB_T(0xb3468f4550192bf7), TO_LIMB_T(0x0bb6cde49d8ba257) }, + { TO_LIMB_T(0x590b62c7ff8a513f), TO_LIMB_T(0x314b4ce372cacefd), + TO_LIMB_T(0x6bef32ce94b8a800), TO_LIMB_T(0x6ddf84a095713d5f), + TO_LIMB_T(0x64eace4cb0982191), TO_LIMB_T(0x0386213c651b888d) }, + { TO_LIMB_T(0xa5310a31111bbcdd), TO_LIMB_T(0xa14ac0f5da148982), + TO_LIMB_T(0xf9ad9cc95423d2e9), TO_LIMB_T(0xaa6ec095283ee4a7), + TO_LIMB_T(0xcf5b1f022e1c9107), TO_LIMB_T(0x01fddf5aed881793) }, + { TO_LIMB_T(0x65a572b0d7a7d950), TO_LIMB_T(0xe25c2d8183473a19), + TO_LIMB_T(0xc2fcebe7cb877dbd), TO_LIMB_T(0x05b2d36c769a89b0), + TO_LIMB_T(0xba12961be86e9efb), TO_LIMB_T(0x07eb1b29c1dfde1f) }, + { TO_LIMB_T(0x93e09572f7c4cd24), TO_LIMB_T(0x364e929076795091), + TO_LIMB_T(0x8569467e68af51b5), TO_LIMB_T(0xa47da89439f5340f), + TO_LIMB_T(0xf4fa918082e44d64), TO_LIMB_T(0x0ad52ba3e6695a79) }, + { TO_LIMB_T(0x911429844e0d5f54), TO_LIMB_T(0xd03f51a3516bb233), + TO_LIMB_T(0x3d587e5640536e66), TO_LIMB_T(0xfa86d2a3a9a73482), + TO_LIMB_T(0xa90ed5adf1ed5537), TO_LIMB_T(0x149c9c326a5e7393) }, + { TO_LIMB_T(0x462bbeb03c12921a), TO_LIMB_T(0xdc9af5fa0a274a17), + TO_LIMB_T(0x9a558ebde836ebed), TO_LIMB_T(0x649ef8f11a4fae46), + TO_LIMB_T(0x8100e1652b3cdc62), TO_LIMB_T(0x1862bd62c291dacb) }, + { TO_LIMB_T(0x05c9b8ca89f12c26), TO_LIMB_T(0x0194160fa9b9ac4f), + TO_LIMB_T(0x6a643d5a6879fa2c), TO_LIMB_T(0x14665bdd8846e19d), + TO_LIMB_T(0xbb1d0d53af3ff6bf), TO_LIMB_T(0x12c7e1c3b28962e5) }, + { TO_LIMB_T(0xb55ebf900b8a3e17), TO_LIMB_T(0xfedc77ec1a9201c4), + TO_LIMB_T(0x1f07db10ea1a4df4), TO_LIMB_T(0x0dfbd15dc41a594d), + TO_LIMB_T(0x389547f2334a5391), TO_LIMB_T(0x02419f98165871a4) }, + { TO_LIMB_T(0xb416af000745fc20), TO_LIMB_T(0x8e563e9d1ea6d0f5), + TO_LIMB_T(0x7c763e17763a0652), TO_LIMB_T(0x01458ef0159ebbef), + TO_LIMB_T(0x8346fe421f96bb13), TO_LIMB_T(0x0d2d7b829ce324d2) }, + { TO_LIMB_T(0x93096bb538d64615), TO_LIMB_T(0x6f2a2619951d823a), + TO_LIMB_T(0x8f66b3ea59514fa4), TO_LIMB_T(0xf563e63704f7092f), + TO_LIMB_T(0x724b136c4cf2d9fa), TO_LIMB_T(0x046959cfcfd0bf49) }, + { TO_LIMB_T(0xea748d4b6e405346), TO_LIMB_T(0x91e9079c2c02d58f), + TO_LIMB_T(0x41064965946d9b59), TO_LIMB_T(0xa06731f1d2bbe1ee), + TO_LIMB_T(0x07f897e267a33f1b), TO_LIMB_T(0x1017290919210e5f) }, + { TO_LIMB_T(0x872aa6c17d985097), TO_LIMB_T(0xeecc53161264562a), + TO_LIMB_T(0x07afe37afff55002), TO_LIMB_T(0x54759078e5be6838), + TO_LIMB_T(0xc4b92d15db8acca8), TO_LIMB_T(0x106d87d1b51d13b9) } + }; + /* ... + * y_den = x'^15 + k_(4,14) * x'^14 + k_(4,13) * x'^13 + ... + k_(4,0) + */ + static const vec384 isogeny_map_y_den[] = { /* (k_(4,*)<<384) % P */ + { TO_LIMB_T(0xeb6c359d47e52b1c), TO_LIMB_T(0x18ef5f8a10634d60), + TO_LIMB_T(0xddfa71a0889d5b7e), TO_LIMB_T(0x723e71dcc5fc1323), + TO_LIMB_T(0x52f45700b70d5c69), TO_LIMB_T(0x0a8b981ee47691f1) }, + { TO_LIMB_T(0x616a3c4f5535b9fb), TO_LIMB_T(0x6f5f037395dbd911), + TO_LIMB_T(0xf25f4cc5e35c65da), TO_LIMB_T(0x3e50dffea3c62658), + TO_LIMB_T(0x6a33dca523560776), TO_LIMB_T(0x0fadeff77b6bfe3e) }, + { TO_LIMB_T(0x2be9b66df470059c), TO_LIMB_T(0x24a2c159a3d36742), + TO_LIMB_T(0x115dbe7ad10c2a37), TO_LIMB_T(0xb6634a652ee5884d), + TO_LIMB_T(0x04fe8bb2b8d81af4), TO_LIMB_T(0x01c2a7a256fe9c41) }, + { TO_LIMB_T(0xf27bf8ef3b75a386), TO_LIMB_T(0x898b367476c9073f), + TO_LIMB_T(0x24482e6b8c2f4e5f), TO_LIMB_T(0xc8e0bbd6fe110806), + TO_LIMB_T(0x59b0c17f7631448a), TO_LIMB_T(0x11037cd58b3dbfbd) }, + { TO_LIMB_T(0x31c7912ea267eec6), TO_LIMB_T(0x1dbf6f1c5fcdb700), + TO_LIMB_T(0xd30d4fe3ba86fdb1), TO_LIMB_T(0x3cae528fbee9a2a4), + TO_LIMB_T(0xb1cce69b6aa9ad9a), TO_LIMB_T(0x044393bb632d94fb) }, + { TO_LIMB_T(0xc66ef6efeeb5c7e8), TO_LIMB_T(0x9824c289dd72bb55), + TO_LIMB_T(0x71b1a4d2f119981d), TO_LIMB_T(0x104fc1aafb0919cc), + TO_LIMB_T(0x0e49df01d942a628), TO_LIMB_T(0x096c3a09773272d4) }, + { TO_LIMB_T(0x9abc11eb5fadeff4), TO_LIMB_T(0x32dca50a885728f0), + TO_LIMB_T(0xfb1fa3721569734c), TO_LIMB_T(0xc4b76271ea6506b3), + TO_LIMB_T(0xd466a75599ce728e), TO_LIMB_T(0x0c81d4645f4cb6ed) }, + { TO_LIMB_T(0x4199f10e5b8be45b), TO_LIMB_T(0xda64e495b1e87930), + TO_LIMB_T(0xcb353efe9b33e4ff), TO_LIMB_T(0x9e9efb24aa6424c6), + TO_LIMB_T(0xf08d33680a237465), TO_LIMB_T(0x0d3378023e4c7406) }, + { TO_LIMB_T(0x7eb4ae92ec74d3a5), TO_LIMB_T(0xc341b4aa9fac3497), + TO_LIMB_T(0x5be603899e907687), TO_LIMB_T(0x03bfd9cca75cbdeb), + TO_LIMB_T(0x564c2935a96bfa93), TO_LIMB_T(0x0ef3c33371e2fdb5) }, + { TO_LIMB_T(0x7ee91fd449f6ac2e), TO_LIMB_T(0xe5d5bd5cb9357a30), + TO_LIMB_T(0x773a8ca5196b1380), TO_LIMB_T(0xd0fda172174ed023), + TO_LIMB_T(0x6cb95e0fa776aead), TO_LIMB_T(0x0d22d5a40cec7cff) }, + { TO_LIMB_T(0xf727e09285fd8519), TO_LIMB_T(0xdc9d55a83017897b), + TO_LIMB_T(0x7549d8bd057894ae), TO_LIMB_T(0x178419613d90d8f8), + TO_LIMB_T(0xfce95ebdeb5b490a), TO_LIMB_T(0x0467ffaef23fc49e) }, + { TO_LIMB_T(0xc1769e6a7c385f1b), TO_LIMB_T(0x79bc930deac01c03), + TO_LIMB_T(0x5461c75a23ede3b5), TO_LIMB_T(0x6e20829e5c230c45), + TO_LIMB_T(0x828e0f1e772a53cd), TO_LIMB_T(0x116aefa749127bff) }, + { TO_LIMB_T(0x101c10bf2744c10a), TO_LIMB_T(0xbbf18d053a6a3154), + TO_LIMB_T(0xa0ecf39ef026f602), TO_LIMB_T(0xfc009d4996dc5153), + TO_LIMB_T(0xb9000209d5bd08d3), TO_LIMB_T(0x189e5fe4470cd73c) }, + { TO_LIMB_T(0x7ebd546ca1575ed2), TO_LIMB_T(0xe47d5a981d081b55), + TO_LIMB_T(0x57b2b625b6d4ca21), TO_LIMB_T(0xb0a1ba04228520cc), + TO_LIMB_T(0x98738983c2107ff3), TO_LIMB_T(0x13dddbc4799d81d6) }, + { TO_LIMB_T(0x09319f2e39834935), TO_LIMB_T(0x039e952cbdb05c21), + TO_LIMB_T(0x55ba77a9a2f76493), TO_LIMB_T(0xfd04e3dfc6086467), + TO_LIMB_T(0xfb95832e7d78742e), TO_LIMB_T(0x0ef9c24eccaf5e0e) } + }; + vec384 Zz_powers[15], map[15], xn, xd, yn, yd; + + /* lay down Z^2 powers in descending order */ + sqr_fp(Zz_powers[14], p->Z); /* ZZ^1 */ +#ifdef __OPTIMIZE_SIZE__ + for (size_t i = 14; i > 0; i--) + mul_fp(Zz_powers[i-1], Zz_powers[i], Zz_powers[14]); +#else + sqr_fp(Zz_powers[13], Zz_powers[14]); /* ZZ^2 1+1 */ + mul_fp(Zz_powers[12], Zz_powers[14], Zz_powers[13]);/* ZZ^3 2+1 */ + sqr_fp(Zz_powers[11], Zz_powers[13]); /* ZZ^4 2+2 */ + mul_fp(Zz_powers[10], Zz_powers[13], Zz_powers[12]);/* ZZ^5 2+3 */ + sqr_fp(Zz_powers[9], Zz_powers[12]); /* ZZ^6 3+3 */ + mul_fp(Zz_powers[8], Zz_powers[12], Zz_powers[11]);/* ZZ^7 3+4 */ + sqr_fp(Zz_powers[7], Zz_powers[11]); /* ZZ^8 4+4 */ + mul_fp(Zz_powers[6], Zz_powers[11], Zz_powers[10]);/* ZZ^9 4+5 */ + sqr_fp(Zz_powers[5], Zz_powers[10]); /* ZZ^10 5+5 */ + mul_fp(Zz_powers[4], Zz_powers[10], Zz_powers[9]); /* ZZ^11 5+6 */ + sqr_fp(Zz_powers[3], Zz_powers[9]); /* ZZ^12 6+6 */ + mul_fp(Zz_powers[2], Zz_powers[9], Zz_powers[8]); /* ZZ^13 6+7 */ + sqr_fp(Zz_powers[1], Zz_powers[8]); /* ZZ^14 7+7 */ + mul_fp(Zz_powers[0], Zz_powers[8], Zz_powers[7]); /* ZZ^15 7+8 */ +#endif + + map_fp_times_Zz(map, isogeny_map_x_num, Zz_powers + 4, 11); + mul_fp(xn, p->X, isogeny_map_x_num[11]); + add_fp(xn, xn, map[10]); + map_fp(xn, p->X, map, 10); + + map_fp_times_Zz(map, isogeny_map_x_den, Zz_powers + 5, 10); + add_fp(xd, p->X, map[9]); + map_fp(xd, p->X, map, 9); + mul_fp(xd, xd, Zz_powers[14]); /* xd *= Z^2 */ + + map_fp_times_Zz(map, isogeny_map_y_num, Zz_powers, 15); + mul_fp(yn, p->X, isogeny_map_y_num[15]); + add_fp(yn, yn, map[14]); + map_fp(yn, p->X, map, 14); + mul_fp(yn, yn, p->Y); /* yn *= Y */ + + map_fp_times_Zz(map, isogeny_map_y_den, Zz_powers, 15); + add_fp(yd, p->X, map[14]); + map_fp(yd, p->X, map, 14); + mul_fp(Zz_powers[14], Zz_powers[14], p->Z); + mul_fp(yd, yd, Zz_powers[14]); /* yd *= Z^3 */ + + /* convert (xn, xd, yn, yd) to Jacobian coordinates */ + mul_fp(out->Z, xd, yd); /* Z = xd * yd */ + mul_fp(out->X, xn, yd); + mul_fp(out->X, out->X, out->Z); /* X = xn * xd * yd^2 */ + sqr_fp(out->Y, out->Z); + mul_fp(out->Y, out->Y, xd); + mul_fp(out->Y, out->Y, yn); /* Y = yn * xd^3 * yd^2 */ +} + +static void map_to_isogenous_E1(POINTonE1 *p, const vec384 u) +{ + static const vec384 minus_A = { /* P - A */ + TO_LIMB_T(0x8a9955f1650a005a), TO_LIMB_T(0x9865b3d192cfe93c), + TO_LIMB_T(0xaed3ed0f3ef3c441), TO_LIMB_T(0x3c962ef33d92c442), + TO_LIMB_T(0x22e438dbd74f94a2), TO_LIMB_T(0x04acbc265478c915) + }; + static const vec384 Z = { /* (11<<384) % P */ + TO_LIMB_T(0x886c00000023ffdc), TO_LIMB_T(0x0f70008d3090001d), + TO_LIMB_T(0x77672417ed5828c3), TO_LIMB_T(0x9dac23e943dc1740), + TO_LIMB_T(0x50553f1b9c131521), TO_LIMB_T(0x078c712fbe0ab6e8) + }; + static const vec384 sqrt_minus_ZZZ = { + TO_LIMB_T(0x43b571cad3215f1f), TO_LIMB_T(0xccb460ef1c702dc2), + TO_LIMB_T(0x742d884f4f97100b), TO_LIMB_T(0xdb2c3e3238a3382b), + TO_LIMB_T(0xe40f3fa13fce8f88), TO_LIMB_T(0x0073a2af9892a2ff) + }; + static const vec384 ZxA = { + TO_LIMB_T(0x7f674ea0a8915178), TO_LIMB_T(0xb0f945fc13b8fa65), + TO_LIMB_T(0x4b46759a38e87d76), TO_LIMB_T(0x2e7a929641bbb6a1), + TO_LIMB_T(0x1668ddfa462bf6b6), TO_LIMB_T(0x00960e2ed1cf294c) + }; + vec384 uu, tv2, x2n, gx1, gxd, y2; +#if 0 + vec384 xn, x1n, xd, y, y1, Zuu, tv4; +#else +# define xn p->X +# define y p->Y +# define xd p->Z +# define x1n xn +# define y1 y +# define Zuu x2n +# define tv4 y1 +#endif +#define sgn0_fp(a) (sgn0_pty_mont_384((a), BLS12_381_P, p0) & 1) + bool_t e1, e2; + + /* + * as per map_to_curve() from poc/sswu_opt.sage at + * https://github.com/cfrg/draft-irtf-cfrg-hash-to-curve + */ + /* x numerator variants */ + sqr_fp(uu, u); /* uu = u^2 */ + mul_fp(Zuu, Z, uu); /* Zuu = Z * uu */ + sqr_fp(tv2, Zuu); /* tv2 = Zuu^2 */ + add_fp(tv2, tv2, Zuu); /* tv2 = tv2 + Zuu */ + add_fp(x1n, tv2, BLS12_381_Rx.p); /* x1n = tv2 + 1 */ + mul_fp(x1n, x1n, Bprime_E1); /* x1n = x1n * B */ + mul_fp(x2n, Zuu, x1n); /* x2n = Zuu * x1n */ + + /* x denumenator */ + mul_fp(xd, minus_A, tv2); /* xd = -A * tv2 */ + e1 = vec_is_zero(xd, sizeof(xd)); /* e1 = xd == 0 */ + vec_select(xd, ZxA, xd, sizeof(xd), e1); /* # If xd == 0, set xd = Z*A */ + + /* y numerators variants */ + sqr_fp(tv2, xd); /* tv2 = xd^2 */ + mul_fp(gxd, xd, tv2); /* gxd = xd^3 */ + mul_fp(tv2, Aprime_E1, tv2); /* tv2 = A * tv2 */ + sqr_fp(gx1, x1n); /* gx1 = x1n^2 */ + add_fp(gx1, gx1, tv2); /* gx1 = gx1 + tv2 # x1n^2 + A*xd^2 */ + mul_fp(gx1, gx1, x1n); /* gx1 = gx1 * x1n # x1n^3 + A*x1n*xd^2 */ + mul_fp(tv2, Bprime_E1, gxd); /* tv2 = B * gxd */ + add_fp(gx1, gx1, tv2); /* gx1 = gx1 + tv2 # x1^3 + A*x1*xd^2 + B*xd^3 */ + sqr_fp(tv4, gxd); /* tv4 = gxd^2 */ + mul_fp(tv2, gx1, gxd); /* tv2 = gx1 * gxd */ + mul_fp(tv4, tv4, tv2); /* tv4 = tv4 * tv2 # gx1*gxd^3 */ + e2 = recip_sqrt_fp(y1, tv4); /* y1 = tv4^c1 # (gx1*gxd^3)^((p-3)/4) */ + mul_fp(y1, y1, tv2); /* y1 = y1 * tv2 # gx1*gxd*y1 */ + mul_fp(y2, y1, sqrt_minus_ZZZ); /* y2 = y1 * c2 # y2 = y1*sqrt(-Z^3) */ + mul_fp(y2, y2, uu); /* y2 = y2 * uu */ + mul_fp(y2, y2, u); /* y2 = y2 * u */ + + /* choose numerators */ + vec_select(xn, x1n, x2n, sizeof(xn), e2); /* xn = e2 ? x1n : x2n */ + vec_select(y, y1, y2, sizeof(y), e2); /* y = e2 ? y1 : y2 */ + + e1 = sgn0_fp(u); + e2 = sgn0_fp(y); + cneg_fp(y, y, e1^e2); /* fix sign of y */ + /* return (xn, xd, y, 1) */ + + /* convert (xn, xd, y, 1) to Jacobian projective coordinates */ + mul_fp(p->X, xn, xd); /* X = xn * xd */ + mul_fp(p->Y, y, gxd); /* Y = y * xd^3 */ +#ifndef xd + vec_copy(p->Z, xd, sizeof(xd)); /* Z = xd */ +#else +# undef xn +# undef y +# undef xd +# undef x1n +# undef y1 +# undef Zuu +# undef tv4 +#endif +#undef sgn0_fp +} + +static void POINTonE1_add_n_dbl(POINTonE1 *out, const POINTonE1 *p, size_t n) +{ + POINTonE1_dadd(out, out, p, NULL); + while(n--) + POINTonE1_double(out, out); +} + +static void POINTonE1_times_minus_z(POINTonE1 *out, const POINTonE1 *in) +{ + POINTonE1_double(out, in); /* 1: 0x2 */ + POINTonE1_add_n_dbl(out, in, 2); /* 2..4: 0x3..0xc */ + POINTonE1_add_n_dbl(out, in, 3); /* 5..8: 0xd..0x68 */ + POINTonE1_add_n_dbl(out, in, 9); /* 9..18: 0x69..0xd200 */ + POINTonE1_add_n_dbl(out, in, 32); /* 19..51: ..0xd20100000000 */ + POINTonE1_add_n_dbl(out, in, 16); /* 52..68: ..0xd201000000010000 */ +} + +/* + * |u|, |v| are expected to be in Montgomery representation + */ +static void map_to_g1(POINTonE1 *out, const vec384 u, const vec384 v) +{ + POINTonE1 p; + + map_to_isogenous_E1(&p, u); + + if (v != NULL) { + map_to_isogenous_E1(out, v); /* borrow |out| */ + POINTonE1_dadd(&p, &p, out, Aprime_E1); + } + + isogeny_map_to_E1(&p, &p); /* sprinkle isogenous powder */ + + /* clear the cofactor by multiplying |p| by 1-z, 0xd201000000010001 */ + POINTonE1_times_minus_z(out, &p); + POINTonE1_dadd(out, out, &p, NULL); +} + +void blst_map_to_g1(POINTonE1 *out, const vec384 u, const vec384 v) +{ map_to_g1(out, u, v); } + +static void Encode_to_G1(POINTonE1 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ + vec384 u[1]; + + hash_to_field(u, 1, aug, aug_len, msg, msg_len, DST, DST_len); + map_to_g1(p, u[0], NULL); +} + +void blst_encode_to_g1(POINTonE1 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ Encode_to_G1(p, msg, msg_len, DST, DST_len, aug, aug_len); } + +static void Hash_to_G1(POINTonE1 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ + vec384 u[2]; + + hash_to_field(u, 2, aug, aug_len, msg, msg_len, DST, DST_len); + map_to_g1(p, u[0], u[1]); +} + +void blst_hash_to_g1(POINTonE1 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ Hash_to_G1(p, msg, msg_len, DST, DST_len, aug, aug_len); } + +static void sigma(POINTonE1 *out, const POINTonE1 *in); + +#if 0 +#ifdef __OPTIMIZE_SIZE__ +static void POINTonE1_times_zz_minus_1_div_by_3(POINTonE1 *out, + const POINTonE1 *in) +{ + static const byte zz_minus_1_div_by_3[] = { + TO_BYTES(0x0000000055555555ULL), TO_BYTES(0x396c8c005555e156) + }; + size_t n = 126-1; + const POINTonE1 *dblin = in; + + while(n--) { + POINTonE1_double(out, dblin); dblin = out; + if (is_bit_set(zz_minus_1_div_by_3, n)) + POINTonE1_dadd(out, out, in, NULL); + } +} +#else +static void POINTonE1_dbl_n_add(POINTonE1 *out, size_t n, const POINTonE1 *p) +{ + while(n--) + POINTonE1_double(out, out); + POINTonE1_dadd(out, out, p, NULL); +} + +static void POINTonE1_times_zz_minus_1_div_by_3(POINTonE1 *out, + const POINTonE1 *in) +{ + POINTonE1 t3, t5, t7, t11, t85; + + POINTonE1_double(&t7, in); /* 2P */ + POINTonE1_dadd(&t3, &t7, in, NULL); /* 3P */ + POINTonE1_dadd(&t5, &t3, &t7, NULL); /* 5P */ + POINTonE1_dadd(&t7, &t5, &t7, NULL); /* 7P */ + POINTonE1_double(&t85, &t5); /* 10P */ + POINTonE1_dadd(&t11, &t85, in, NULL); /* 11P */ + POINTonE1_dbl_n_add(&t85, 3, &t5); /* 0x55P */ + /* (-0xd201000000010000^2 - 1) / 3 */ + POINTonE1_double(out, &t7); /* 0xe */ + POINTonE1_dbl_n_add(out, 5, &t11); /* 0x1cb */ + POINTonE1_dbl_n_add(out, 3, &t3); /* 0xe5b */ + POINTonE1_dbl_n_add(out, 3, in); /* 0x72d9 */ + POINTonE1_dbl_n_add(out, 5, &t3); /* 0xe5b23 */ + POINTonE1_dbl_n_add(out, 18, &t85); /* 0x396c8c0055 */ + POINTonE1_dbl_n_add(out, 8, &t85); /* 0x396c8c005555 */ + POINTonE1_dbl_n_add(out, 3, &t7); /* 0x1cb646002aaaf */ + POINTonE1_dbl_n_add(out, 7, &t5); /* 0xe5b23001555785 */ + POINTonE1_dbl_n_add(out, 5, &t11); /* 0x1cb646002aaaf0ab */ + POINTonE1_dbl_n_add(out, 41, &t85); /* 0x396c8c005555e1560000000055 */ + POINTonE1_dbl_n_add(out, 8, &t85); /* 0x396c8c005555e156000000005555 */ + POINTonE1_dbl_n_add(out, 8, &t85); /* 0x396c8c005555e15600000000555555 */ + POINTonE1_dbl_n_add(out, 8, &t85); /* 0x396c8c005555e1560000000055555555 */ +} +#endif + +static bool_t POINTonE1_in_G1(const POINTonE1 *P) +{ + POINTonE1 t0, t1, t2; + + /* Bowe, S., "Faster subgroup checks for BLS12-381" */ + sigma(&t0, P); /* σ(P) */ + sigma(&t1, &t0); /* σ²(P) */ + + POINTonE1_double(&t0, &t0); /* 2σ(P) */ + POINTonE1_dadd(&t2, &t1, P, NULL); /* P + σ²(P) */ + POINTonE1_cneg(&t2, 1); /* - P - σ²(P) */ + POINTonE1_dadd(&t2, &t2, &t0, NULL); /* 2σ(P) - P - σ²(P) */ + POINTonE1_times_zz_minus_1_div_by_3( &t0, &t2); + POINTonE1_cneg(&t1, 1); + POINTonE1_dadd(&t0, &t0, &t1, NULL); /* [(z²-1)/3](2σ(P) - P - σ²(P)) */ + /* - σ²(P) */ + return vec_is_zero(t0.Z, sizeof(t0.Z)); +} +#else +static bool_t POINTonE1_in_G1(const POINTonE1 *P) +{ + POINTonE1 t0, t1; + + /* Scott, M., https://eprint.iacr.org/2021/1130 */ + POINTonE1_times_minus_z(&t0, P); + POINTonE1_times_minus_z(&t1, &t0); + POINTonE1_cneg(&t1, 1); /* [-z²]P */ + + sigma(&t0, P); /* σ(P) */ + sigma(&t0, &t0); /* σ²(P) */ + + return POINTonE1_is_equal(&t0, &t1); +} +#endif + +int blst_p1_in_g1(const POINTonE1 *p) +{ return (int)POINTonE1_in_G1(p); } + +int blst_p1_affine_in_g1(const POINTonE1_affine *p) +{ + POINTonE1 P; + + vec_copy(P.X, p->X, 2*sizeof(P.X)); + vec_select(P.Z, p->X, BLS12_381_Rx.p, sizeof(P.Z), + vec_is_zero(p, sizeof(*p))); + + return (int)POINTonE1_in_G1(&P); +} diff --git a/crypto/blst_src/map_to_g2.c b/crypto/blst_src/map_to_g2.c new file mode 100644 index 00000000000..90fd86e9d31 --- /dev/null +++ b/crypto/blst_src/map_to_g2.c @@ -0,0 +1,444 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "point.h" +#include "fields.h" + +/* + * y^2 = x^3 + A'*x + B', isogenous one + */ +static const vec384x Aprime_E2 = { /* 240*i */ + { 0 }, + { TO_LIMB_T(0xe53a000003135242), TO_LIMB_T(0x01080c0fdef80285), + TO_LIMB_T(0xe7889edbe340f6bd), TO_LIMB_T(0x0b51375126310601), + TO_LIMB_T(0x02d6985717c744ab), TO_LIMB_T(0x1220b4e979ea5467) } +}; +static const vec384x Bprime_E2 = { /* 1012 + 1012*i */ + { TO_LIMB_T(0x22ea00000cf89db2), TO_LIMB_T(0x6ec832df71380aa4), + TO_LIMB_T(0x6e1b94403db5a66e), TO_LIMB_T(0x75bf3c53a79473ba), + TO_LIMB_T(0x3dd3a569412c0a34), TO_LIMB_T(0x125cdb5e74dc4fd1) }, + { TO_LIMB_T(0x22ea00000cf89db2), TO_LIMB_T(0x6ec832df71380aa4), + TO_LIMB_T(0x6e1b94403db5a66e), TO_LIMB_T(0x75bf3c53a79473ba), + TO_LIMB_T(0x3dd3a569412c0a34), TO_LIMB_T(0x125cdb5e74dc4fd1) } +}; + +static void map_fp2_times_Zz(vec384x map[], const vec384x isogeny_map[], + const vec384x Zz_powers[], size_t n) +{ + while (n--) + mul_fp2(map[n], isogeny_map[n], Zz_powers[n]); +} + +static void map_fp2(vec384x acc, const vec384x x, const vec384x map[], size_t n) +{ + while (n--) { + mul_fp2(acc, acc, x); + add_fp2(acc, acc, map[n]); + } +} + +static void isogeny_map_to_E2(POINTonE2 *out, const POINTonE2 *p) +{ + /* + * x = x_num / x_den, where + * x_num = k_(1,3) * x'^3 + k_(1,2) * x'^2 + k_(1,1) * x' + k_(1,0) + * ... + */ + static const vec384x isogeny_map_x_num[] = { /* (k_(1,*)<<384) % P */ + {{ TO_LIMB_T(0x47f671c71ce05e62), TO_LIMB_T(0x06dd57071206393e), + TO_LIMB_T(0x7c80cd2af3fd71a2), TO_LIMB_T(0x048103ea9e6cd062), + TO_LIMB_T(0xc54516acc8d037f6), TO_LIMB_T(0x13808f550920ea41) }, + { TO_LIMB_T(0x47f671c71ce05e62), TO_LIMB_T(0x06dd57071206393e), + TO_LIMB_T(0x7c80cd2af3fd71a2), TO_LIMB_T(0x048103ea9e6cd062), + TO_LIMB_T(0xc54516acc8d037f6), TO_LIMB_T(0x13808f550920ea41) }}, + {{ 0 }, + { TO_LIMB_T(0x5fe55555554c71d0), TO_LIMB_T(0x873fffdd236aaaa3), + TO_LIMB_T(0x6a6b4619b26ef918), TO_LIMB_T(0x21c2888408874945), + TO_LIMB_T(0x2836cda7028cabc5), TO_LIMB_T(0x0ac73310a7fd5abd) }}, + {{ TO_LIMB_T(0x0a0c5555555971c3), TO_LIMB_T(0xdb0c00101f9eaaae), + TO_LIMB_T(0xb1fb2f941d797997), TO_LIMB_T(0xd3960742ef416e1c), + TO_LIMB_T(0xb70040e2c20556f4), TO_LIMB_T(0x149d7861e581393b) }, + { TO_LIMB_T(0xaff2aaaaaaa638e8), TO_LIMB_T(0x439fffee91b55551), + TO_LIMB_T(0xb535a30cd9377c8c), TO_LIMB_T(0x90e144420443a4a2), + TO_LIMB_T(0x941b66d3814655e2), TO_LIMB_T(0x0563998853fead5e) }}, + {{ TO_LIMB_T(0x40aac71c71c725ed), TO_LIMB_T(0x190955557a84e38e), + TO_LIMB_T(0xd817050a8f41abc3), TO_LIMB_T(0xd86485d4c87f6fb1), + TO_LIMB_T(0x696eb479f885d059), TO_LIMB_T(0x198e1a74328002d2) }, + { 0 }} + }; + /* ... + * x_den = x'^2 + k_(2,1) * x' + k_(2,0) + */ + static const vec384x isogeny_map_x_den[] = { /* (k_(2,*)<<384) % P */ + {{ 0 }, + { TO_LIMB_T(0x1f3affffff13ab97), TO_LIMB_T(0xf25bfc611da3ff3e), + TO_LIMB_T(0xca3757cb3819b208), TO_LIMB_T(0x3e6427366f8cec18), + TO_LIMB_T(0x03977bc86095b089), TO_LIMB_T(0x04f69db13f39a952) }}, + {{ TO_LIMB_T(0x447600000027552e), TO_LIMB_T(0xdcb8009a43480020), + TO_LIMB_T(0x6f7ee9ce4a6e8b59), TO_LIMB_T(0xb10330b7c0a95bc6), + TO_LIMB_T(0x6140b1fcfb1e54b7), TO_LIMB_T(0x0381be097f0bb4e1) }, + { TO_LIMB_T(0x7588ffffffd8557d), TO_LIMB_T(0x41f3ff646e0bffdf), + TO_LIMB_T(0xf7b1e8d2ac426aca), TO_LIMB_T(0xb3741acd32dbb6f8), + TO_LIMB_T(0xe9daf5b9482d581f), TO_LIMB_T(0x167f53e0ba7431b8) }} + }; + /* + * y = y' * y_num / y_den, where + * y_num = k_(3,3) * x'^3 + k_(3,2) * x'^2 + k_(3,1) * x' + k_(3,0) + * ... + */ + static const vec384x isogeny_map_y_num[] = { /* (k_(3,*)<<384) % P */ + {{ TO_LIMB_T(0x96d8f684bdfc77be), TO_LIMB_T(0xb530e4f43b66d0e2), + TO_LIMB_T(0x184a88ff379652fd), TO_LIMB_T(0x57cb23ecfae804e1), + TO_LIMB_T(0x0fd2e39eada3eba9), TO_LIMB_T(0x08c8055e31c5d5c3) }, + { TO_LIMB_T(0x96d8f684bdfc77be), TO_LIMB_T(0xb530e4f43b66d0e2), + TO_LIMB_T(0x184a88ff379652fd), TO_LIMB_T(0x57cb23ecfae804e1), + TO_LIMB_T(0x0fd2e39eada3eba9), TO_LIMB_T(0x08c8055e31c5d5c3) }}, + {{ 0 }, + { TO_LIMB_T(0xbf0a71c71c91b406), TO_LIMB_T(0x4d6d55d28b7638fd), + TO_LIMB_T(0x9d82f98e5f205aee), TO_LIMB_T(0xa27aa27b1d1a18d5), + TO_LIMB_T(0x02c3b2b2d2938e86), TO_LIMB_T(0x0c7d13420b09807f) }}, + {{ TO_LIMB_T(0xd7f9555555531c74), TO_LIMB_T(0x21cffff748daaaa8), + TO_LIMB_T(0x5a9ad1866c9bbe46), TO_LIMB_T(0x4870a2210221d251), + TO_LIMB_T(0x4a0db369c0a32af1), TO_LIMB_T(0x02b1ccc429ff56af) }, + { TO_LIMB_T(0xe205aaaaaaac8e37), TO_LIMB_T(0xfcdc000768795556), + TO_LIMB_T(0x0c96011a8a1537dd), TO_LIMB_T(0x1c06a963f163406e), + TO_LIMB_T(0x010df44c82a881e6), TO_LIMB_T(0x174f45260f808feb) }}, + {{ TO_LIMB_T(0xa470bda12f67f35c), TO_LIMB_T(0xc0fe38e23327b425), + TO_LIMB_T(0xc9d3d0f2c6f0678d), TO_LIMB_T(0x1c55c9935b5a982e), + TO_LIMB_T(0x27f6c0e2f0746764), TO_LIMB_T(0x117c5e6e28aa9054) }, + { 0 }} + }; + /* ... + * y_den = x'^3 + k_(4,2) * x'^2 + k_(4,1) * x' + k_(4,0) + */ + static const vec384x isogeny_map_y_den[] = { /* (k_(4,*)<<384) % P */ + {{ TO_LIMB_T(0x0162fffffa765adf), TO_LIMB_T(0x8f7bea480083fb75), + TO_LIMB_T(0x561b3c2259e93611), TO_LIMB_T(0x11e19fc1a9c875d5), + TO_LIMB_T(0xca713efc00367660), TO_LIMB_T(0x03c6a03d41da1151) }, + { TO_LIMB_T(0x0162fffffa765adf), TO_LIMB_T(0x8f7bea480083fb75), + TO_LIMB_T(0x561b3c2259e93611), TO_LIMB_T(0x11e19fc1a9c875d5), + TO_LIMB_T(0xca713efc00367660), TO_LIMB_T(0x03c6a03d41da1151) }}, + {{ 0 }, + { TO_LIMB_T(0x5db0fffffd3b02c5), TO_LIMB_T(0xd713f52358ebfdba), + TO_LIMB_T(0x5ea60761a84d161a), TO_LIMB_T(0xbb2c75a34ea6c44a), + TO_LIMB_T(0x0ac6735921c1119b), TO_LIMB_T(0x0ee3d913bdacfbf6) }}, + {{ TO_LIMB_T(0x66b10000003affc5), TO_LIMB_T(0xcb1400e764ec0030), + TO_LIMB_T(0xa73e5eb56fa5d106), TO_LIMB_T(0x8984c913a0fe09a9), + TO_LIMB_T(0x11e10afb78ad7f13), TO_LIMB_T(0x05429d0e3e918f52) }, + { TO_LIMB_T(0x534dffffffc4aae6), TO_LIMB_T(0x5397ff174c67ffcf), + TO_LIMB_T(0xbff273eb870b251d), TO_LIMB_T(0xdaf2827152870915), + TO_LIMB_T(0x393a9cbaca9e2dc3), TO_LIMB_T(0x14be74dbfaee5748) }} + }; + vec384x Zz_powers[3], map[3], xn, xd, yn, yd; + + /* lay down Z^2 powers in descending order */ + sqr_fp2(Zz_powers[2], p->Z); /* ZZ^1 */ + sqr_fp2(Zz_powers[1], Zz_powers[2]); /* ZZ^2 1+1 */ + mul_fp2(Zz_powers[0], Zz_powers[2], Zz_powers[1]); /* ZZ^3 2+1 */ + + map_fp2_times_Zz(map, isogeny_map_x_num, Zz_powers, 3); + mul_fp2(xn, p->X, isogeny_map_x_num[3]); + add_fp2(xn, xn, map[2]); + map_fp2(xn, p->X, map, 2); + + map_fp2_times_Zz(map, isogeny_map_x_den, Zz_powers + 1, 2); + add_fp2(xd, p->X, map[1]); + map_fp2(xd, p->X, map, 1); + mul_fp2(xd, xd, Zz_powers[2]); /* xd *= Z^2 */ + + map_fp2_times_Zz(map, isogeny_map_y_num, Zz_powers, 3); + mul_fp2(yn, p->X, isogeny_map_y_num[3]); + add_fp2(yn, yn, map[2]); + map_fp2(yn, p->X, map, 2); + mul_fp2(yn, yn, p->Y); /* yn *= Y */ + + map_fp2_times_Zz(map, isogeny_map_y_den, Zz_powers, 3); + add_fp2(yd, p->X, map[2]); + map_fp2(yd, p->X, map, 2); + mul_fp2(Zz_powers[2], Zz_powers[2], p->Z); + mul_fp2(yd, yd, Zz_powers[2]); /* yd *= Z^3 */ + + /* convert (xn, xd, yn, yd) to Jacobian coordinates */ + mul_fp2(out->Z, xd, yd); /* Z = xd * yd */ + mul_fp2(out->X, xn, yd); + mul_fp2(out->X, out->X, out->Z); /* X = xn * xd * yd^2 */ + sqr_fp2(out->Y, out->Z); + mul_fp2(out->Y, out->Y, xd); + mul_fp2(out->Y, out->Y, yn); /* Y = yn * xd^3 * yd^2 */ +} + +static void map_to_isogenous_E2(POINTonE2 *p, const vec384x u) +{ + static const vec384x minus_A = { + { 0 }, + { TO_LIMB_T(0xd4c4fffffcec5869), TO_LIMB_T(0x1da3f3eed25bfd79), + TO_LIMB_T(0x7fa833c5136fff67), TO_LIMB_T(0x59261433cd540cbd), + TO_LIMB_T(0x48450f5f2b84682c), TO_LIMB_T(0x07e05d00bf959233) } + }; + static const vec384x Z = { /* -2 - i */ + { TO_LIMB_T(0x87ebfffffff9555c), TO_LIMB_T(0x656fffe5da8ffffa), + TO_LIMB_T(0x0fd0749345d33ad2), TO_LIMB_T(0xd951e663066576f4), + TO_LIMB_T(0xde291a3d41e980d3), TO_LIMB_T(0x0815664c7dfe040d) }, + { TO_LIMB_T(0x43f5fffffffcaaae), TO_LIMB_T(0x32b7fff2ed47fffd), + TO_LIMB_T(0x07e83a49a2e99d69), TO_LIMB_T(0xeca8f3318332bb7a), + TO_LIMB_T(0xef148d1ea0f4c069), TO_LIMB_T(0x040ab3263eff0206) } + }; + static const vec384x recip_ZZZ = { /* 1/(Z^3) */ + { TO_LIMB_T(0x65018f5c28f598eb), TO_LIMB_T(0xe6020417f022d916), + TO_LIMB_T(0xd6327313288369c7), TO_LIMB_T(0x622ded8eb447156f), + TO_LIMB_T(0xe52a2aee72c2a01f), TO_LIMB_T(0x089812fb8481ffe4) }, + { TO_LIMB_T(0x2574eb851eb8619f), TO_LIMB_T(0xdba2e97912925604), + TO_LIMB_T(0x67e495a909e7a18e), TO_LIMB_T(0xdf2da23b8145b8f7), + TO_LIMB_T(0xcf5d3728310ebf6d), TO_LIMB_T(0x11be446236f4c116) } + }; + static const vec384x magic_ZZZ = { /* 1/Z^3 = a + b*i */ + /* a^2 + b^2 */ + { TO_LIMB_T(0xaa7eb851eb8508e0), TO_LIMB_T(0x1c54fdf360989374), + TO_LIMB_T(0xc87f2fc6e716c62e), TO_LIMB_T(0x0124aefb1f9efea7), + TO_LIMB_T(0xb2f8be63e844865c), TO_LIMB_T(0x08b47f775a7ef35a) }, + /* (a^2 + b^2)^((P-3)/4) */ + { TO_LIMB_T(0xe4132bbd838cf70a), TO_LIMB_T(0x01d769ac83772c19), + TO_LIMB_T(0xa83dd6e974c22e45), TO_LIMB_T(0xbc8ec3e777b08dff), + TO_LIMB_T(0xc035c2042ecf5da3), TO_LIMB_T(0x073929e97f0850bf) } + }; + static const vec384x ZxA = { /* 240 - 480*i */ + { TO_LIMB_T(0xe53a000003135242), TO_LIMB_T(0x01080c0fdef80285), + TO_LIMB_T(0xe7889edbe340f6bd), TO_LIMB_T(0x0b51375126310601), + TO_LIMB_T(0x02d6985717c744ab), TO_LIMB_T(0x1220b4e979ea5467) }, + { TO_LIMB_T(0xa989fffff9d8b0d2), TO_LIMB_T(0x3b47e7dda4b7faf3), + TO_LIMB_T(0xff50678a26dffece), TO_LIMB_T(0xb24c28679aa8197a), + TO_LIMB_T(0x908a1ebe5708d058), TO_LIMB_T(0x0fc0ba017f2b2466) } + }; + vec384x uu, tv2, tv4, x2n, gx1, gxd, y2; +#if 0 + vec384x xn, x1n, xd, y, y1, Zuu; +#else +# define xn p->X +# define y p->Y +# define xd p->Z +# define x1n xn +# define y1 y +# define Zuu x2n +#endif +#define sgn0_fp2(a) (sgn0_pty_mont_384x((a), BLS12_381_P, p0) & 1) + bool_t e1, e2; + + /* + * as per map_to_curve() from poc/sswu_opt.sage at + * https://github.com/cfrg/draft-irtf-cfrg-hash-to-curve + * with 9mod16 twists... + */ + /* x numerator variants */ + sqr_fp2(uu, u); /* uu = u^2 */ + mul_fp2(Zuu, Z, uu); /* Zuu = Z * uu */ + sqr_fp2(tv2, Zuu); /* tv2 = Zuu^2 */ + add_fp2(tv2, tv2, Zuu); /* tv2 = tv2 + Zuu */ + add_fp2(x1n, tv2, BLS12_381_Rx.p2); /* x1n = tv2 + 1 */ + mul_fp2(x1n, x1n, Bprime_E2); /* x1n = x1n * B */ + mul_fp2(x2n, Zuu, x1n); /* x2n = Zuu * x1n */ + + /* x denumenator */ + mul_fp2(xd, minus_A, tv2); /* xd = -A * tv2 */ + e1 = vec_is_zero(xd, sizeof(xd)); /* e1 = xd == 0 */ + vec_select(xd, ZxA, xd, sizeof(xd), e1); /* # If xd == 0, set xd = Z*A */ + + /* y numerators variants */ + sqr_fp2(tv2, xd); /* tv2 = xd^2 */ + mul_fp2(gxd, xd, tv2); /* gxd = xd^3 */ + mul_fp2(tv2, Aprime_E2, tv2); /* tv2 = A * tv2 */ + sqr_fp2(gx1, x1n); /* gx1 = x1n^2 */ + add_fp2(gx1, gx1, tv2); /* gx1 = gx1 + tv2 # x1n^2 + A*xd^2 */ + mul_fp2(gx1, gx1, x1n); /* gx1 = gx1 * x1n # x1n^3 + A*x1n*xd^2 */ + mul_fp2(tv2, Bprime_E2, gxd); /* tv2 = B * gxd */ + add_fp2(gx1, gx1, tv2); /* gx1 = gx1 + tv2 # x1^3 + A*x1*xd^2 + B*xd^3 */ + sqr_fp2(tv4, gxd); /* tv4 = gxd^2 */ + mul_fp2(tv2, gx1, gxd); /* tv2 = gx1 * gxd */ + mul_fp2(tv4, tv4, tv2); /* tv4 = tv4 * tv2 # gx1*gxd^3 */ + e2 = recip_sqrt_fp2(y1, tv4, /* y1 = tv4^c1 # (gx1*gxd^3)^((p^2-9)/16) */ + recip_ZZZ, magic_ZZZ); + mul_fp2(y1, y1, tv2); /* y1 = y1 * tv2 # gx1*gxd*y1 */ + mul_fp2(y2, y1, uu); /* y2 = y1 * uu */ + mul_fp2(y2, y2, u); /* y2 = y2 * u */ + + /* choose numerators */ + vec_select(xn, x1n, x2n, sizeof(xn), e2); /* xn = e2 ? x1n : x2n */ + vec_select(y, y1, y2, sizeof(y), e2); /* y = e2 ? y1 : y2 */ + + e1 = sgn0_fp2(u); + e2 = sgn0_fp2(y); + cneg_fp2(y, y, e1^e2); /* fix sign of y */ + /* return (xn, xd, y, 1) */ + + /* convert (xn, xd, y, 1) to Jacobian projective coordinates */ + mul_fp2(p->X, xn, xd); /* X = xn * xd */ + mul_fp2(p->Y, y, gxd); /* Y = y * xd^3 */ +#ifndef xd + vec_copy(p->Z, xd, sizeof(xd)); /* Z = xd */ +#else +# undef xn +# undef y +# undef xd +# undef x1n +# undef y1 +# undef Zuu +# undef tv4 +#endif +#undef sgn0_fp2 +} + +#if 0 +static const byte h_eff[] = { + TO_BYTES(0xe8020005aaa95551), TO_BYTES(0x59894c0adebbf6b4), + TO_BYTES(0xe954cbc06689f6a3), TO_BYTES(0x2ec0ec69d7477c1a), + TO_BYTES(0x6d82bf015d1212b0), TO_BYTES(0x329c2f178731db95), + TO_BYTES(0x9986ff031508ffe1), TO_BYTES(0x88e2a8e9145ad768), + TO_BYTES(0x584c6a0ea91b3528), TO_BYTES(0x0bc69f08f2ee75b3) +}; + +static void clear_cofactor(POINTonE2 *out, const POINTonE2 *p) +{ POINTonE2_mult_w5(out, p, h_eff, 636); } +#else +/* + * As per suggestions in "7. Clearing the cofactor" at + * https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 + */ +static void POINTonE2_add_n_dbl(POINTonE2 *out, const POINTonE2 *p, size_t n) +{ + POINTonE2_dadd(out, out, p, NULL); + while(n--) + POINTonE2_double(out, out); +} + +static void POINTonE2_times_minus_z(POINTonE2 *out, const POINTonE2 *in) +{ + POINTonE2_double(out, in); /* 1: 0x2 */ + POINTonE2_add_n_dbl(out, in, 2); /* 2..4: 0x3..0xc */ + POINTonE2_add_n_dbl(out, in, 3); /* 5..8: 0xd..0x68 */ + POINTonE2_add_n_dbl(out, in, 9); /* 9..18: 0x69..0xd200 */ + POINTonE2_add_n_dbl(out, in, 32); /* 19..51: ..0xd20100000000 */ + POINTonE2_add_n_dbl(out, in, 16); /* 52..68: ..0xd201000000010000 */ +} + +static void psi(POINTonE2 *out, const POINTonE2 *in); + +static void clear_cofactor(POINTonE2 *out, const POINTonE2 *p) +{ + POINTonE2 t0, t1; + + /* A.Budroni, F.Pintore, "Efficient hash maps to G2 on BLS curves" */ + POINTonE2_double(out, p); /* out = 2P */ + psi(out, out); /* out = Ψ(2P) */ + psi(out, out); /* out = Ψ²(2P) */ + + vec_copy(&t0, p, sizeof(t0)); + POINTonE2_cneg(&t0, 1); /* t0 = -P */ + psi(&t1, &t0); /* t1 = -Ψ(P) */ + POINTonE2_dadd(out, out, &t0, NULL);/* out = Ψ²(2P) - P */ + POINTonE2_dadd(out, out, &t1, NULL);/* out = Ψ²(2P) - P - Ψ(P) */ + + POINTonE2_times_minus_z(&t0, p); /* t0 = [-z]P */ + POINTonE2_dadd(&t0, &t0, p, NULL); /* t0 = [-z + 1]P */ + POINTonE2_dadd(&t0, &t0, &t1, NULL);/* t0 = [-z + 1]P - Ψ(P) */ + POINTonE2_times_minus_z(&t1, &t0); /* t1 = [z² - z]P + [z]Ψ(P) */ + POINTonE2_dadd(out, out, &t1, NULL);/* out = [z² - z - 1]P */ + /* + [z - 1]Ψ(P) */ + /* + Ψ²(2P) */ +} +#endif + +/* + * |u|, |v| are expected to be in Montgomery representation + */ +static void map_to_g2(POINTonE2 *out, const vec384x u, const vec384x v) +{ + POINTonE2 p; + + map_to_isogenous_E2(&p, u); + + if (v != NULL) { + map_to_isogenous_E2(out, v); /* borrow |out| */ + POINTonE2_dadd(&p, &p, out, Aprime_E2); + } + + isogeny_map_to_E2(&p, &p); /* sprinkle isogenous powder */ + clear_cofactor(out, &p); +} + +void blst_map_to_g2(POINTonE2 *out, const vec384x u, const vec384x v) +{ map_to_g2(out, u, v); } + +static void Encode_to_G2(POINTonE2 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ + vec384x u[1]; + + hash_to_field(u[0], 2, aug, aug_len, msg, msg_len, DST, DST_len); + map_to_g2(p, u[0], NULL); +} + +void blst_encode_to_g2(POINTonE2 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ Encode_to_G2(p, msg, msg_len, DST, DST_len, aug, aug_len); } + +static void Hash_to_G2(POINTonE2 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ + vec384x u[2]; + + hash_to_field(u[0], 4, aug, aug_len, msg, msg_len, DST, DST_len); + map_to_g2(p, u[0], u[1]); +} + +void blst_hash_to_g2(POINTonE2 *p, const unsigned char *msg, size_t msg_len, + const unsigned char *DST, size_t DST_len, + const unsigned char *aug, size_t aug_len) +{ Hash_to_G2(p, msg, msg_len, DST, DST_len, aug, aug_len); } + +static bool_t POINTonE2_in_G2(const POINTonE2 *P) +{ +#if 0 + POINTonE2 t0, t1, t2; + + /* Bowe, S., "Faster subgroup checks for BLS12-381" */ + psi(&t0, P); /* Ψ(P) */ + psi(&t0, &t0); /* Ψ²(P) */ + psi(&t1, &t0); /* Ψ³(P) */ + + POINTonE2_times_minus_z(&t2, &t1); + POINTonE2_dadd(&t0, &t0, &t2, NULL); + POINTonE2_cneg(&t0, 1); + POINTonE2_dadd(&t0, &t0, P, NULL); /* [z]Ψ³(P) - Ψ²(P) + P */ + + return vec_is_zero(t0.Z, sizeof(t0.Z)); +#else + POINTonE2 t0, t1; + + /* Scott, M., https://eprint.iacr.org/2021/1130 */ + psi(&t0, P); /* Ψ(P) */ + + POINTonE2_times_minus_z(&t1, P); + POINTonE2_cneg(&t1, 1); /* [z]P */ + + return POINTonE2_is_equal(&t0, &t1); +#endif +} + +int blst_p2_in_g2(const POINTonE2 *p) +{ return (int)POINTonE2_in_G2(p); } + +int blst_p2_affine_in_g2(const POINTonE2_affine *p) +{ + POINTonE2 P; + + vec_copy(P.X, p->X, 2*sizeof(P.X)); + vec_select(P.Z, p->X, BLS12_381_Rx.p, sizeof(P.Z), + vec_is_zero(p, sizeof(*p))); + + return (int)POINTonE2_in_G2(&P); +} diff --git a/crypto/blst_src/multi_scalar.c b/crypto/blst_src/multi_scalar.c new file mode 100644 index 00000000000..d0b3deefe25 --- /dev/null +++ b/crypto/blst_src/multi_scalar.c @@ -0,0 +1,414 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" +#include "point.h" + +/* + * Infinite point among inputs would be devastating. Shall we change it? + */ +#define POINTS_TO_AFFINE_IMPL(prefix, ptype, bits, field) \ +static void ptype##s_to_affine(ptype##_affine dst[], \ + const ptype *const points[], size_t npoints) \ +{ \ + size_t i; \ + vec##bits *acc, ZZ, ZZZ; \ + const ptype *point = NULL; \ + const size_t stride = sizeof(ptype)==sizeof(POINTonE1) ? 1536 : 768; \ +\ + while (npoints) { \ + const ptype *p, *const *walkback; \ + size_t delta = stride<npoints ? stride : npoints; \ +\ + point = *points ? *points++ : point+1; \ + acc = (vec##bits *)dst; \ + vec_copy(acc++, point->Z, sizeof(vec##bits)); \ + for (i = 1; i < delta; i++, acc++) \ + point = *points ? *points++ : point+1, \ + mul_##field(acc[0], acc[-1], point->Z); \ +\ + --acc; reciprocal_##field(acc[0], acc[0]); \ +\ + walkback = points-1, p = point, --delta, dst += delta; \ + for (i = 0; i < delta; i++, acc--, dst--) { \ + mul_##field(acc[-1], acc[-1], acc[0]); /* 1/Z */\ + sqr_##field(ZZ, acc[-1]); /* 1/Z^2 */\ + mul_##field(ZZZ, ZZ, acc[-1]); /* 1/Z^3 */\ + mul_##field(acc[-1], p->Z, acc[0]); \ + mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ + mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + p = (p == *walkback) ? *--walkback : p-1; \ + } \ + sqr_##field(ZZ, acc[0]); /* 1/Z^2 */\ + mul_##field(ZZZ, ZZ, acc[0]); /* 1/Z^3 */\ + mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ + mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + ++delta, dst += delta, npoints -= delta; \ + } \ +} \ +\ +void prefix##s_to_affine(ptype##_affine dst[], const ptype *const points[], \ + size_t npoints) \ +{ ptype##s_to_affine(dst, points, npoints); } + +POINTS_TO_AFFINE_IMPL(blst_p1, POINTonE1, 384, fp) +POINTS_TO_AFFINE_IMPL(blst_p2, POINTonE2, 384x, fp2) + +/* + * This is two-step multi-scalar multiplication procedure. First, given + * a set of points you pre-compute a table for chosen windowing factor + * [expressed in bits with value between 2 and 14], and then you pass + * this table to the actual multiplication procedure along with scalars. + * Idea is that the pre-computed table will be reused multiple times. In + * which case multiplication runs faster than below Pippenger algorithm + * implementation for up to ~16K points for wbits=8, naturally at the + * expense of multi-megabyte table. One can trade even more memory for + * performance, but each wbits increment doubles the memory requirement, + * so at some point it gets prohibively large... For reference, without + * reusing the table it's faster than Pippenger algorithm for up ~32 + * points [with wbits=5]... + */ + +#define SCRATCH_SZ(ptype) (sizeof(ptype)==sizeof(POINTonE1) ? 8192 : 4096) + +#define PRECOMPUTE_WBITS_IMPL(prefix, ptype, bits, field, one) \ +static void ptype##_precompute_row_wbits(ptype row[], size_t wbits, \ + const ptype##_affine *point) \ +{ \ + size_t i, j, n = (size_t)1 << (wbits-1); \ + /* row[-1] is implicit infinity */\ + vec_copy(&row[0], point, sizeof(*point)); /* row[0]=p*1 */\ + vec_copy(&row[0].Z, one, sizeof(row[0].Z)); \ + ptype##_double(&row[1], &row[0]); /* row[1]=p*(1+1) */\ + for (i = 2, j = 1; i < n; i += 2, j++) \ + ptype##_add_affine(&row[i], &row[i-1], point), /* row[2]=p*(2+1) */\ + ptype##_double(&row[i+1], &row[j]); /* row[3]=p*(2+2) */\ +} /* row[4] ... */\ +\ +static void ptype##s_to_affine_row_wbits(ptype##_affine dst[], ptype src[], \ + size_t wbits, size_t npoints) \ +{ \ + size_t total = npoints << (wbits-1); \ + size_t nwin = (size_t)1 << (wbits-1); \ + size_t i, j; \ + vec##bits *acc, ZZ, ZZZ; \ +\ + src += total; \ + acc = (vec##bits *)src; \ + vec_copy(acc++, one, sizeof(vec##bits)); \ + for (i = 0; i < npoints; i++) \ + for (j = nwin; --src, --j; acc++) \ + mul_##field(acc[0], acc[-1], src->Z); \ +\ + --acc; reciprocal_##field(acc[0], acc[0]); \ +\ + for (i = 0; i < npoints; i++) { \ + vec_copy(dst++, src++, sizeof(ptype##_affine)); \ + for (j = 1; j < nwin; j++, acc--, src++, dst++) { \ + mul_##field(acc[-1], acc[-1], acc[0]); /* 1/Z */\ + sqr_##field(ZZ, acc[-1]); /* 1/Z^2 */\ + mul_##field(ZZZ, ZZ, acc[-1]); /* 1/Z^3 */\ + mul_##field(acc[-1], src->Z, acc[0]); \ + mul_##field(dst->X, src->X, ZZ); /* X = X'/Z^2 */\ + mul_##field(dst->Y, src->Y, ZZZ); /* Y = Y'/Z^3 */\ + } \ + } \ +} \ +\ +/* flat |points[n]| can be placed at the end of |table[n<<(wbits-1)]| */\ +static void ptype##s_precompute_wbits(ptype##_affine table[], size_t wbits, \ + const ptype##_affine *const points[], \ + size_t npoints) \ +{ \ + size_t total = npoints << (wbits-1); \ + size_t nwin = (size_t)1 << (wbits-1); \ + size_t nmin = wbits>9 ? (size_t)1: (size_t)1 << (9-wbits); \ + size_t i, top = 0; \ + ptype *rows, *row; \ + const ptype##_affine *point = NULL; \ + size_t stride = ((512*1024)/sizeof(ptype##_affine)) >> wbits; \ + if (stride == 0) stride = 1; \ +\ + while (npoints >= nmin) { \ + size_t limit = total - npoints; \ +\ + if (top + (stride << wbits) > limit) { \ + stride = (limit - top) >> wbits; \ + if (stride == 0) break; \ + } \ + rows = row = (ptype *)(&table[top]); \ + for (i = 0; i < stride; i++, row += nwin) \ + point = *points ? *points++ : point+1, \ + ptype##_precompute_row_wbits(row, wbits, point); \ + ptype##s_to_affine_row_wbits(&table[top], rows, wbits, stride); \ + top += stride << (wbits-1); \ + npoints -= stride; \ + } \ + rows = row = alloca(2*sizeof(ptype##_affine) * npoints * nwin); \ + for (i = 0; i < npoints; i++, row += nwin) \ + point = *points ? *points++ : point+1, \ + ptype##_precompute_row_wbits(row, wbits, point); \ + ptype##s_to_affine_row_wbits(&table[top], rows, wbits, npoints); \ +} \ +\ +size_t prefix##s_mult_wbits_precompute_sizeof(size_t wbits, size_t npoints) \ +{ return (sizeof(ptype##_affine)*npoints) << (wbits-1); } \ +void prefix##s_mult_wbits_precompute(ptype##_affine table[], size_t wbits, \ + const ptype##_affine *const points[], \ + size_t npoints) \ +{ ptype##s_precompute_wbits(table, wbits, points, npoints); } + +#define POINTS_MULT_WBITS_IMPL(prefix, ptype, bits, field, one) \ +static void ptype##_gather_booth_wbits(ptype *p, const ptype##_affine row[], \ + size_t wbits, limb_t booth_idx) \ +{ \ + bool_t booth_sign = (booth_idx >> wbits) & 1; \ + bool_t idx_is_zero; \ + static const ptype##_affine infinity = { 0 }; \ +\ + booth_idx &= ((limb_t)1 << wbits) - 1; \ + idx_is_zero = is_zero(booth_idx); \ + booth_idx -= 1 ^ idx_is_zero; \ + vec_select(p, &infinity, &row[booth_idx], sizeof(row[0]), idx_is_zero); \ + ptype##_cneg(p, booth_sign); \ +} \ +\ +static void ptype##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ + size_t wbits, size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype scratch[]) \ +{ \ + limb_t wmask, wval; \ + size_t i, j, z, nbytes, window, nwin = (size_t)1 << (wbits-1); \ + const byte *scalar, *const *scalar_s = scalars; \ + const ptype##_affine *row = table; \ +\ + size_t scratch_sz = SCRATCH_SZ(ptype); \ + if (scratch == NULL) { \ + scratch_sz /= 4; /* limit to 288K */ \ + scratch_sz = scratch_sz < npoints ? scratch_sz : npoints; \ + scratch = alloca(sizeof(ptype) * scratch_sz); \ + } \ +\ + nbytes = (nbits + 7)/8; /* convert |nbits| to bytes */ \ + scalar = *scalar_s++; \ +\ + /* top excess bits modulo target window size */ \ + window = nbits % wbits; /* yes, it may be zero */ \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ +\ + nbits -= window; \ + z = is_zero(nbits); \ + wval = (get_wval_limb(scalar, nbits - (z^1), wbits + (z^1)) << z) & wmask; \ + wval = booth_encode(wval, wbits); \ + ptype##_gather_booth_wbits(&scratch[0], row, wbits, wval); \ + row += nwin; \ +\ + i = 1; vec_zero(ret, sizeof(*ret)); \ + while (nbits > 0) { \ + for (j = i; i < npoints; i++, j++, row += nwin) { \ + if (j == scratch_sz) \ + ptype##s_accumulate(ret, scratch, j), j = 0; \ + scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ + wval = get_wval_limb(scalar, nbits - 1, window + 1) & wmask; \ + wval = booth_encode(wval, wbits); \ + ptype##_gather_booth_wbits(&scratch[j], row, wbits, wval); \ + } \ + ptype##s_accumulate(ret, scratch, j); \ +\ + for (j = 0; j < wbits; j++) \ + ptype##_double(ret, ret); \ +\ + window = wbits; \ + wmask = ((limb_t)1 << (window + 1)) - 1; \ + nbits -= window; \ + i = 0; row = table; scalar_s = scalars; \ + } \ +\ + for (j = i; i < npoints; i++, j++, row += nwin) { \ + if (j == scratch_sz) \ + ptype##s_accumulate(ret, scratch, j), j = 0; \ + scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ + wval = (get_wval_limb(scalar, 0, wbits) << 1) & wmask; \ + wval = booth_encode(wval, wbits); \ + ptype##_gather_booth_wbits(&scratch[j], row, wbits, wval); \ + } \ + ptype##s_accumulate(ret, scratch, j); \ +} \ +\ +size_t prefix##s_mult_wbits_scratch_sizeof(size_t npoints) \ +{ \ + const size_t scratch_sz = SCRATCH_SZ(ptype); \ + return sizeof(ptype) * (npoints < scratch_sz ? npoints : scratch_sz); \ +} \ +void prefix##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ + size_t wbits, size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype scratch[]) \ +{ ptype##s_mult_wbits(ret, table, wbits, npoints, scalars, nbits, scratch); } + +PRECOMPUTE_WBITS_IMPL(blst_p1, POINTonE1, 384, fp, BLS12_381_Rx.p) +POINTS_MULT_WBITS_IMPL(blst_p1, POINTonE1, 384, fp, BLS12_381_Rx.p) + +PRECOMPUTE_WBITS_IMPL(blst_p2, POINTonE2, 384x, fp2, BLS12_381_Rx.p2) +POINTS_MULT_WBITS_IMPL(blst_p2, POINTonE2, 384x, fp2, BLS12_381_Rx.p2) + +/* + * Pippenger algorithm implementation, fastest option for larger amount + * of points... + */ + +static size_t pippenger_window_size(size_t npoints) +{ + size_t wbits; + + for (wbits=0; npoints>>=1; wbits++) ; + + return wbits>12 ? wbits-3 : (wbits>4 ? wbits-2 : (wbits ? 2 : 1)); +} + +#define DECLARE_PRIVATE_POINTXYZZ(ptype, bits) \ +typedef struct { vec##bits X,Y,ZZZ,ZZ; } ptype##xyzz; + +#define POINTS_MULT_PIPPENGER_IMPL(prefix, ptype) \ +static void ptype##_integrate_buckets(ptype *out, ptype##xyzz buckets[], \ + size_t wbits) \ +{ \ + ptype##xyzz ret[1], acc[1]; \ + size_t n = (size_t)1 << wbits; \ +\ + /* Calculate sum of x[i-1]*i for i=1 through 1<<|wbits|. */\ + vec_copy(acc, &buckets[--n], sizeof(acc)); \ + vec_copy(ret, &buckets[n], sizeof(ret)); \ + vec_zero(&buckets[n], sizeof(buckets[n])); \ + while (n--) { \ + ptype##xyzz_dadd(acc, acc, &buckets[n]); \ + ptype##xyzz_dadd(ret, ret, acc); \ + vec_zero(&buckets[n], sizeof(buckets[n])); \ + } \ + ptype##xyzz_to_Jacobian(out, ret); \ +} \ +\ +static void ptype##_bucket(ptype##xyzz buckets[], limb_t booth_idx, \ + size_t wbits, const ptype##_affine *p) \ +{ \ + bool_t booth_sign = (booth_idx >> wbits) & 1; \ +\ + booth_idx &= (1<<wbits) - 1; \ + if (booth_idx--) \ + ptype##xyzz_dadd_affine(&buckets[booth_idx], &buckets[booth_idx], \ + p, booth_sign); \ +} \ +\ +static void ptype##_prefetch(const ptype##xyzz buckets[], limb_t booth_idx, \ + size_t wbits) \ +{ \ + booth_idx &= (1<<wbits) - 1; \ + if (booth_idx--) \ + vec_prefetch(&buckets[booth_idx], sizeof(buckets[booth_idx])); \ +} \ +\ +static void ptype##s_tile_pippenger(ptype *ret, \ + const ptype##_affine *const points[], \ + size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype##xyzz buckets[], \ + size_t bit0, size_t wbits, size_t cbits) \ +{ \ + limb_t wmask, wval, wnxt; \ + size_t i, z, nbytes; \ + const byte *scalar = *scalars++; \ + const ptype##_affine *point = *points++; \ +\ + nbytes = (nbits + 7)/8; /* convert |nbits| to bytes */ \ + wmask = ((limb_t)1 << (wbits+1)) - 1; \ + z = is_zero(bit0); \ + bit0 -= z^1; wbits += z^1; \ + wval = (get_wval_limb(scalar, bit0, wbits) << z) & wmask; \ + wval = booth_encode(wval, cbits); \ + scalar = *scalars ? *scalars++ : scalar+nbytes; \ + wnxt = (get_wval_limb(scalar, bit0, wbits) << z) & wmask; \ + wnxt = booth_encode(wnxt, cbits); \ + npoints--; /* account for prefetch */ \ +\ + ptype##_bucket(buckets, wval, cbits, point); \ + for (i = 1; i < npoints; i++) { \ + wval = wnxt; \ + scalar = *scalars ? *scalars++ : scalar+nbytes; \ + wnxt = (get_wval_limb(scalar, bit0, wbits) << z) & wmask; \ + wnxt = booth_encode(wnxt, cbits); \ + ptype##_prefetch(buckets, wnxt, cbits); \ + point = *points ? *points++ : point+1; \ + ptype##_bucket(buckets, wval, cbits, point); \ + } \ + point = *points ? *points++ : point+1; \ + ptype##_bucket(buckets, wnxt, cbits, point); \ + ptype##_integrate_buckets(ret, buckets, cbits - 1); \ +} \ +\ +static void ptype##s_mult_pippenger(ptype *ret, \ + const ptype##_affine *const points[], \ + size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype##xyzz buckets[], size_t window) \ +{ \ + size_t i, wbits, cbits, bit0 = nbits; \ + ptype tile[1]; \ +\ + window = window ? window : pippenger_window_size(npoints); \ + vec_zero(buckets, sizeof(buckets[0]) << (window-1)); \ + vec_zero(ret, sizeof(*ret)); \ +\ + /* top excess bits modulo target window size */ \ + wbits = nbits % window; /* yes, it may be zero */ \ + cbits = wbits + 1; \ + while (bit0 -= wbits) { \ + ptype##s_tile_pippenger(tile, points, npoints, scalars, nbits, \ + buckets, bit0, wbits, cbits); \ + ptype##_dadd(ret, ret, tile, NULL); \ + for (i = 0; i < window; i++) \ + ptype##_double(ret, ret); \ + cbits = wbits = window; \ + } \ + ptype##s_tile_pippenger(tile, points, npoints, scalars, nbits, \ + buckets, 0, wbits, cbits); \ + ptype##_dadd(ret, ret, tile, NULL); \ +} \ +\ +size_t prefix##s_mult_pippenger_scratch_sizeof(size_t npoints) \ +{ return sizeof(ptype##xyzz) << (pippenger_window_size(npoints)-1); } \ +void prefix##s_tile_pippenger(ptype *ret, \ + const ptype##_affine *const points[], \ + size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype##xyzz scratch[], \ + size_t bit0, size_t window) \ +{ \ + size_t wbits, cbits; \ +\ + if (bit0 + window > nbits) wbits = nbits - bit0, cbits = wbits + 1; \ + else wbits = cbits = window; \ + ptype##s_tile_pippenger(ret, points, npoints, scalars, nbits, scratch, \ + bit0, wbits, cbits); \ +} \ +void prefix##s_mult_pippenger(ptype *ret, \ + const ptype##_affine *const points[], \ + size_t npoints, \ + const byte *const scalars[], size_t nbits, \ + ptype##xyzz scratch[]) \ +{ ptype##s_mult_pippenger(ret, points, npoints, scalars, nbits, scratch, 0); } + +DECLARE_PRIVATE_POINTXYZZ(POINTonE1, 384) +POINTXYZZ_TO_JACOBIAN_IMPL(POINTonE1, 384, fp) +POINTXYZZ_DADD_IMPL(POINTonE1, 384, fp) +POINTXYZZ_DADD_AFFINE_IMPL(POINTonE1, 384, fp, BLS12_381_Rx.p) +POINTS_MULT_PIPPENGER_IMPL(blst_p1, POINTonE1) + +DECLARE_PRIVATE_POINTXYZZ(POINTonE2, 384x) +POINTXYZZ_TO_JACOBIAN_IMPL(POINTonE2, 384x, fp2) +POINTXYZZ_DADD_IMPL(POINTonE2, 384x, fp2) +POINTXYZZ_DADD_AFFINE_IMPL(POINTonE2, 384x, fp2, BLS12_381_Rx.p2) +POINTS_MULT_PIPPENGER_IMPL(blst_p2, POINTonE2) diff --git a/crypto/blst_src/no_asm.h b/crypto/blst_src/no_asm.h new file mode 100644 index 00000000000..be7bf47e197 --- /dev/null +++ b/crypto/blst_src/no_asm.h @@ -0,0 +1,1345 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#if LIMB_T_BITS==32 +typedef unsigned long long llimb_t; +#endif + +#if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 || defined(__STDC_NO_VLA__) +# error "unsupported compiler" +#endif + +#if defined(__clang__) +# pragma GCC diagnostic ignored "-Wstatic-in-inline" +#endif + +#if !defined(__clang__) && !defined(__builtin_assume) +# if defined(__GNUC__) && __GNUC__>=5 +# define __builtin_assume(condition) if (!(condition)) __builtin_unreachable() +# elif defined(_MSC_VER) +# define __builtin_assume(condition) __assume(condition) +# else +# define __builtin_assume(condition) (void)(condition) +# endif +#endif + +static void mul_mont_n(limb_t ret[], const limb_t a[], const limb_t b[], + const limb_t p[], limb_t n0, size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t mask, borrow, mx, hi, tmp[n+1], carry; + size_t i, j; + + for (mx=b[0], hi=0, i=0; i<n; i++) { + limbx = (mx * (llimb_t)a[i]) + hi; + tmp[i] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + mx = n0*tmp[0]; + tmp[i] = hi; + + for (carry=0, j=0; ; ) { + limbx = (mx * (llimb_t)p[0]) + tmp[0]; + hi = (limb_t)(limbx >> LIMB_T_BITS); + for (i=1; i<n; i++) { + limbx = (mx * (llimb_t)p[i] + hi) + tmp[i]; + tmp[i-1] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + limbx = tmp[i] + (hi + (llimb_t)carry); + tmp[i-1] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + + if (++j==n) + break; + + for (mx=b[j], hi=0, i=0; i<n; i++) { + limbx = (mx * (llimb_t)a[i] + hi) + tmp[i]; + tmp[i] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + mx = n0*tmp[0]; + limbx = hi + (llimb_t)carry; + tmp[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); +} + +#define MUL_MONT_IMPL(bits) \ +inline void mul_mont_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits b, const vec##bits p, limb_t n0) \ +{ mul_mont_n(ret, a, b, p, n0, NLIMBS(bits)); } \ +\ +inline void sqr_mont_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits p, limb_t n0) \ +{ mul_mont_n(ret, a, a, p, n0, NLIMBS(bits)); } + +/* + * 256-bit subroutines can handle arbitrary modulus, even non-"sparse", + * but we have to harmonize the naming with assembly. + */ +#define mul_mont_256 mul_mont_sparse_256 +#define sqr_mont_256 sqr_mont_sparse_256 +MUL_MONT_IMPL(256) +#undef mul_mont_256 +#undef sqr_mont_256 +MUL_MONT_IMPL(384) + +static void add_mod_n(limb_t ret[], const limb_t a[], const limb_t b[], + const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t mask, carry, borrow, tmp[n]; + size_t i; + + for (carry=0, i=0; i<n; i++) { + limbx = a[i] + (b[i] + (llimb_t)carry); + tmp[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); +} + +#define ADD_MOD_IMPL(bits) \ +inline void add_mod_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits b, const vec##bits p) \ +{ add_mod_n(ret, a, b, p, NLIMBS(bits)); } + +ADD_MOD_IMPL(256) +ADD_MOD_IMPL(384) + +static void sub_mod_n(limb_t ret[], const limb_t a[], const limb_t b[], + const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t mask, carry, borrow; + size_t i; + + for (borrow=0, i=0; i<n; i++) { + limbx = a[i] - (b[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = 0 - borrow; + launder(mask); + + for (carry=0, i=0; i<n; i++) { + limbx = ret[i] + ((p[i] & mask) + (llimb_t)carry); + ret[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } +} + +#define SUB_MOD_IMPL(bits) \ +inline void sub_mod_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits b, const vec##bits p) \ +{ sub_mod_n(ret, a, b, p, NLIMBS(bits)); } + +SUB_MOD_IMPL(256) +SUB_MOD_IMPL(384) + +static void mul_by_3_mod_n(limb_t ret[], const limb_t a[], const limb_t p[], + size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t mask, carry, borrow, tmp[n], two_a[n]; + size_t i; + + for (carry=0, i=0; i<n; i++) { + limb_t a_i = a[i]; + tmp[i] = a_i<<1 | carry; + carry = a_i>>(LIMB_T_BITS-1); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + two_a[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + two_a[i] = (two_a[i] & ~mask) | (tmp[i] & mask); + + for (carry=0, i=0; i<n; i++) { + limbx = a[i] + (two_a[i] + (llimb_t)carry); + tmp[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); +} + +#define MUL_BY_3_MOD_IMPL(bits) \ +inline void mul_by_3_mod_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits p) \ +{ mul_by_3_mod_n(ret, a, p, NLIMBS(bits)); } + +MUL_BY_3_MOD_IMPL(256) +MUL_BY_3_MOD_IMPL(384) + +static void lshift_mod_n(limb_t ret[], const limb_t a[], size_t count, + const limb_t p[], size_t n) +{ + __builtin_assume(count != 0); + __builtin_assume(n != 0); + llimb_t limbx; + limb_t mask, carry, borrow, tmp[n]; + size_t i; + + while (count--) { + for (carry=0, i=0; i<n; i++) { + limb_t a_i = a[i]; + tmp[i] = a_i<<1 | carry; + carry = a_i>>(LIMB_T_BITS-1); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); + + a = ret; + } +} + +#define LSHIFT_MOD_IMPL(bits) \ +inline void lshift_mod_##bits(vec##bits ret, const vec##bits a, size_t count, \ + const vec##bits p) \ +{ lshift_mod_n(ret, a, count, p, NLIMBS(bits)); } + +LSHIFT_MOD_IMPL(256) +LSHIFT_MOD_IMPL(384) + +static void cneg_mod_n(limb_t ret[], const limb_t a[], bool_t flag, + const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t borrow, mask, tmp[n]; + size_t i; + + for (borrow=0, i=0; i<n; i++) { + limbx = p[i] - (a[i] + (llimb_t)borrow); + tmp[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + flag &= vec_is_zero(a, sizeof(tmp)) ^ 1; + mask = (limb_t)0 - flag; + + for(i=0; i<n; i++) + ret[i] = (a[i] & ~mask) | (tmp[i] & mask); +} + +#define CNEG_MOD_IMPL(bits) \ +inline void cneg_mod_##bits(vec##bits ret, const vec##bits a, bool_t flag, \ + const vec##bits p) \ +{ cneg_mod_n(ret, a, flag, p, NLIMBS(bits)); } + +CNEG_MOD_IMPL(256) +CNEG_MOD_IMPL(384) + +static limb_t check_mod_n(const byte a[], const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t borrow, ai, acc; + size_t i, j; + + for (acc=borrow=0, i=0; i<n; i++) { + for (ai=0, j=0; j<8*sizeof(limb_t); j+=8) + ai |= (limb_t)(*a++) << j; + acc |= ai; + limbx = ai - (p[i] + (llimb_t)borrow); + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + return borrow & (is_zero(acc) ^ 1); +} + +#define CHECK_MOD_IMPL(bits) \ +inline limb_t check_mod_##bits(const pow##bits a, const vec##bits p) \ +{ return check_mod_n(a, p, NLIMBS(bits)); } + +CHECK_MOD_IMPL(256) + +static limb_t add_n_check_mod_n(byte ret[], const byte a[], const byte b[], + const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + limb_t ret_[n], a_[n], b_[n], zero; + + limbs_from_le_bytes(a_, a, sizeof(a_)); + limbs_from_le_bytes(b_, b, sizeof(b_)); + + add_mod_n(ret_, a_, b_, p, n); + zero = vec_is_zero(ret_, sizeof(ret_)); + + le_bytes_from_limbs(ret, ret_, sizeof(ret_)); + + return zero^1; +} + +#define ADD_N_CHECK_MOD_IMPL(bits) \ +inline limb_t add_n_check_mod_##bits(pow##bits ret, const pow##bits a, \ + const pow##bits b, const vec##bits p) \ +{ return add_n_check_mod_n(ret, a, b, p, NLIMBS(bits)); } + +ADD_N_CHECK_MOD_IMPL(256) + +static limb_t sub_n_check_mod_n(byte ret[], const byte a[], const byte b[], + const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + limb_t ret_[n], a_[n], b_[n], zero; + + limbs_from_le_bytes(a_, a, sizeof(a_)); + limbs_from_le_bytes(b_, b, sizeof(b_)); + + sub_mod_n(ret_, a_, b_, p, n); + zero = vec_is_zero(ret_, sizeof(ret_)); + + le_bytes_from_limbs(ret, ret_, sizeof(ret_)); + + return zero^1; +} + +#define SUB_N_CHECK_MOD_IMPL(bits) \ +inline limb_t sub_n_check_mod_##bits(pow##bits ret, const pow##bits a, \ + const pow##bits b, const vec##bits p) \ +{ return sub_n_check_mod_n(ret, a, b, p, NLIMBS(bits)); } + +SUB_N_CHECK_MOD_IMPL(256) + +static void from_mont_n(limb_t ret[], const limb_t a[], + const limb_t p[], limb_t n0, size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t mask, borrow, mx, hi, tmp[n]; + size_t i, j; + + for (j=0; j<n; j++) { + mx = n0*a[0]; + limbx = (mx * (llimb_t)p[0]) + a[0]; + hi = (limb_t)(limbx >> LIMB_T_BITS); + for (i=1; i<n; i++) { + limbx = (mx * (llimb_t)p[i] + hi) + a[i]; + tmp[i-1] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + tmp[i-1] = hi; + a = tmp; + } + + /* this is needed only if input can be non-fully-reduced */ + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = 0 - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); +} + +#define FROM_MONT_IMPL(bits) \ +inline void from_mont_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits p, limb_t n0) \ +{ from_mont_n(ret, a, p, n0, NLIMBS(bits)); } + +FROM_MONT_IMPL(256) +FROM_MONT_IMPL(384) + +static void redc_mont_n(limb_t ret[], const limb_t a[], + const limb_t p[], limb_t n0, size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t mask, carry, borrow, mx, hi, tmp[n]; + const limb_t *b = a; + size_t i, j; + + for (j=0; j<n; j++) { + mx = n0*b[0]; + limbx = (mx * (llimb_t)p[0]) + b[0]; + hi = (limb_t)(limbx >> LIMB_T_BITS); + for (i=1; i<n; i++) { + limbx = (mx * (llimb_t)p[i] + hi) + b[i]; + tmp[i-1] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + tmp[i-1] = hi; + b = tmp; + } + + for (carry=0, i=0; i<n; i++) { + limbx = a[n+i] + (tmp[i] + (llimb_t)carry); + tmp[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + ret[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = carry - borrow; + launder(mask); + + for(i=0; i<n; i++) + ret[i] = (ret[i] & ~mask) | (tmp[i] & mask); +} + +#define REDC_MONT_IMPL(bits, bits2) \ +inline void redc_mont_##bits(vec##bits ret, const vec##bits2 a, \ + const vec##bits p, limb_t n0) \ +{ redc_mont_n(ret, a, p, n0, NLIMBS(bits)); } + +REDC_MONT_IMPL(256, 512) +REDC_MONT_IMPL(384, 768) + +static void rshift_mod_n(limb_t ret[], const limb_t a[], size_t count, + const limb_t p[], size_t n) +{ + __builtin_assume(count != 0); + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t mask, carry, limb, next; + size_t i; + + while (count--) { + mask = 0 - (a[0] & 1); + launder(mask); + for (carry=0, i=0; i<n; i++) { + limbx = a[i] + ((p[i]&mask) + (llimb_t)carry); + ret[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + for (next=ret[0], i=0; i<n-1; i++) { + limb = next >> 1; + next = ret[i+1]; + ret[i] = limb | next << (LIMB_T_BITS-1); + } + ret[i] = next >> 1 | carry << (LIMB_T_BITS-1); + + a = ret; + } +} + +#define RSHIFT_MOD_IMPL(bits) \ +inline void rshift_mod_##bits(vec##bits ret, const vec##bits a, size_t count, \ + const vec##bits p) \ +{ rshift_mod_n(ret, a, count, p, NLIMBS(bits)); } + +RSHIFT_MOD_IMPL(256) +RSHIFT_MOD_IMPL(384) + +#define DIV_BY_2_MOD_IMPL(bits) \ +inline void div_by_2_mod_##bits(vec##bits ret, const vec##bits a, \ + const vec##bits p) \ +{ rshift_mod_n(ret, a, 1, p, NLIMBS(bits)); } + +DIV_BY_2_MOD_IMPL(384) + +static limb_t sgn0_pty_mod_n(const limb_t a[], const limb_t p[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t carry, borrow, ret, tmp[n]; + size_t i; + + ret = a[0] & 1; /* parity */ + + for (carry=0, i=0; i<n; i++) { + limb_t a_i = a[i]; + tmp[i] = a_i<<1 | carry; + carry = a_i>>(LIMB_T_BITS-1); + } + + for (borrow=0, i=0; i<n; i++) { + limbx = tmp[i] - (p[i] + (llimb_t)borrow); + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + ret |= ((carry - borrow) & 2) ^ 2; + + return ret; +} + +inline limb_t sgn0_pty_mod_384(const vec384 a, const vec384 p) +{ return sgn0_pty_mod_n(a, p, NLIMBS(384)); } + +inline limb_t sgn0_pty_mont_384(const vec384 a, const vec384 p, limb_t n0) +{ + vec384 tmp; + + from_mont_n(tmp, a, p, n0, NLIMBS(384)); + + return sgn0_pty_mod_n(tmp, p, NLIMBS(384)); +} + +inline limb_t sgn0_pty_mod_384x(const vec384x a, const vec384 p) +{ + limb_t re, im, sign, prty; + + re = sgn0_pty_mod_n(a[0], p, NLIMBS(384)); + im = sgn0_pty_mod_n(a[1], p, NLIMBS(384)); + + /* a->im!=0 ? sgn0(a->im) : sgn0(a->re) */ + sign = (limb_t)0 - vec_is_zero(a[1], sizeof(vec384)); + sign = (re & sign) | (im & ~sign); + + /* a->re==0 ? prty(a->im) : prty(a->re) */ + prty = (limb_t)0 - vec_is_zero(a[0], sizeof(vec384)); + prty = (im & prty) | (re & ~prty); + + return (sign & 2) | (prty & 1); +} + +inline limb_t sgn0_pty_mont_384x(const vec384x a, const vec384 p, limb_t n0) +{ + vec384x tmp; + + from_mont_n(tmp[0], a[0], p, n0, NLIMBS(384)); + from_mont_n(tmp[1], a[1], p, n0, NLIMBS(384)); + + return sgn0_pty_mod_384x(tmp, p); +} + +void mul_mont_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 p, limb_t n0) +{ + vec384 aa, bb, cc; + + add_mod_n(aa, a[0], a[1], p, NLIMBS(384)); + add_mod_n(bb, b[0], b[1], p, NLIMBS(384)); + mul_mont_n(bb, bb, aa, p, n0, NLIMBS(384)); + mul_mont_n(aa, a[0], b[0], p, n0, NLIMBS(384)); + mul_mont_n(cc, a[1], b[1], p, n0, NLIMBS(384)); + sub_mod_n(ret[0], aa, cc, p, NLIMBS(384)); + sub_mod_n(ret[1], bb, aa, p, NLIMBS(384)); + sub_mod_n(ret[1], ret[1], cc, p, NLIMBS(384)); +} + +/* + * mul_mont_n without final conditional subtraction, which implies + * that modulus is one bit short, which in turn means that there are + * no carries to handle between iterations... + */ +static void mul_mont_nonred_n(limb_t ret[], const limb_t a[], const limb_t b[], + const limb_t p[], limb_t n0, size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t mx, hi, tmp[n+1]; + size_t i, j; + + for (mx=b[0], hi=0, i=0; i<n; i++) { + limbx = (mx * (llimb_t)a[i]) + hi; + tmp[i] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + mx = n0*tmp[0]; + tmp[i] = hi; + + for (j=0; ; ) { + limbx = (mx * (llimb_t)p[0]) + tmp[0]; + hi = (limb_t)(limbx >> LIMB_T_BITS); + for (i=1; i<n; i++) { + limbx = (mx * (llimb_t)p[i] + hi) + tmp[i]; + tmp[i-1] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + tmp[i-1] = tmp[i] + hi; + + if (++j==n) + break; + + for (mx=b[j], hi=0, i=0; i<n; i++) { + limbx = (mx * (llimb_t)a[i] + hi) + tmp[i]; + tmp[i] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + mx = n0*tmp[0]; + tmp[i] = hi; + } + + vec_copy(ret, tmp, sizeof(tmp)-sizeof(limb_t)); +} + +void sqr_n_mul_mont_383(vec384 ret, const vec384 a, size_t count, + const vec384 p, limb_t n0, const vec384 b) +{ + __builtin_assume(count != 0); + while(count--) { + mul_mont_nonred_n(ret, a, a, p, n0, NLIMBS(384)); + a = ret; + } + mul_mont_n(ret, ret, b, p, n0, NLIMBS(384)); +} + +void sqr_mont_382x(vec384x ret, const vec384x a, + const vec384 p, limb_t n0) +{ + llimb_t limbx; + limb_t mask, carry, borrow; + size_t i; + vec384 t0, t1; + + /* "add_mod_n(t0, a[0], a[1], p, NLIMBS(384));" */ + for (carry=0, i=0; i<NLIMBS(384); i++) { + limbx = a[0][i] + (a[1][i] + (llimb_t)carry); + t0[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + /* "sub_mod_n(t1, a[0], a[1], p, NLIMBS(384));" */ + for (borrow=0, i=0; i<NLIMBS(384); i++) { + limbx = a[0][i] - (a[1][i] + (llimb_t)borrow); + t1[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + mask = 0 - borrow; + launder(mask); + + /* "mul_mont_n(ret[1], a[0], a[1], p, n0, NLIMBS(384));" */ + mul_mont_nonred_n(ret[1], a[0], a[1], p, n0, NLIMBS(384)); + + /* "add_mod_n(ret[1], ret[1], ret[1], p, NLIMBS(384));" */ + for (carry=0, i=0; i<NLIMBS(384); i++) { + limb_t a_i = ret[1][i]; + ret[1][i] = a_i<<1 | carry; + carry = a_i>>(LIMB_T_BITS-1); + } + + /* "mul_mont_n(ret[0], t0, t1, p, n0, NLIMBS(384));" */ + mul_mont_nonred_n(ret[0], t0, t1, p, n0, NLIMBS(384)); + + /* account for t1's sign... */ + for (borrow=0, i=0; i<NLIMBS(384); i++) { + limbx = ret[0][i] - ((t0[i] & mask) + (llimb_t)borrow); + ret[0][i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + mask = 0 - borrow; + launder(mask); + for (carry=0, i=0; i<NLIMBS(384); i++) { + limbx = ret[0][i] + ((p[i] & mask) + (llimb_t)carry); + ret[0][i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } +} + +#if defined(__GNUC__) || defined(__clang__) +# define MSB(x) ({ limb_t ret = (x) >> (LIMB_T_BITS-1); launder(ret); ret; }) +#else +# define MSB(x) ((x) >> (LIMB_T_BITS-1)) +#endif + +static size_t num_bits(limb_t l) +{ + limb_t x, mask; + size_t bits = is_zero(l) ^ 1; + + if (sizeof(limb_t) == 8) { + x = l >> (32 & (8*sizeof(limb_t)-1)); + mask = 0 - MSB(0 - x); + bits += 32 & mask; + l ^= (x ^ l) & mask; + } + + x = l >> 16; + mask = 0 - MSB(0 - x); + bits += 16 & mask; + l ^= (x ^ l) & mask; + + x = l >> 8; + mask = 0 - MSB(0 - x); + bits += 8 & mask; + l ^= (x ^ l) & mask; + + x = l >> 4; + mask = 0 - MSB(0 - x); + bits += 4 & mask; + l ^= (x ^ l) & mask; + + x = l >> 2; + mask = 0 - MSB(0 - x); + bits += 2 & mask; + l ^= (x ^ l) & mask; + + bits += l >> 1; + + return bits; +} + +#if defined(__clang_major__) && __clang_major__>7 +__attribute__((optnone)) +#endif +static limb_t lshift_2(limb_t hi, limb_t lo, size_t l) +{ + size_t r = LIMB_T_BITS - l; + limb_t mask = 0 - (is_zero(l)^1); + return (hi << (l&(LIMB_T_BITS-1))) | ((lo & mask) >> (r&(LIMB_T_BITS-1))); +} + +/* + * https://eprint.iacr.org/2020/972 with 'k' being LIMB_T_BITS-1. + */ +static void ab_approximation_n(limb_t a_[2], const limb_t a[], + limb_t b_[2], const limb_t b[], size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + limb_t a_hi, a_lo, b_hi, b_lo, mask; + size_t i; + + i = n-1; + a_hi = a[i], a_lo = a[i-1]; + b_hi = b[i], b_lo = b[i-1]; + for (i--; --i;) { + mask = 0 - is_zero(a_hi | b_hi); + a_hi = ((a_lo ^ a_hi) & mask) ^ a_hi; + b_hi = ((b_lo ^ b_hi) & mask) ^ b_hi; + a_lo = ((a[i] ^ a_lo) & mask) ^ a_lo; + b_lo = ((b[i] ^ b_lo) & mask) ^ b_lo; + } + i = LIMB_T_BITS - num_bits(a_hi | b_hi); + /* |i| can be LIMB_T_BITS if all a[2..]|b[2..] were zeros */ + + a_[0] = a[0], a_[1] = lshift_2(a_hi, a_lo, i); + b_[0] = b[0], b_[1] = lshift_2(b_hi, b_lo, i); +} + +typedef struct { limb_t f0, g0, f1, g1; } factors; + +static void inner_loop_n(factors *fg, const limb_t a_[2], const limb_t b_[2], + size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t f0 = 1, g0 = 0, f1 = 0, g1 = 1; + limb_t a_lo, a_hi, b_lo, b_hi, t_lo, t_hi, odd, borrow, xorm; + + a_lo = a_[0], a_hi = a_[1]; + b_lo = b_[0], b_hi = b_[1]; + + while(n--) { + odd = 0 - (a_lo&1); + + /* a_ -= b_ if a_ is odd */ + t_lo = a_lo, t_hi = a_hi; + limbx = a_lo - (llimb_t)(b_lo & odd); + a_lo = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + limbx = a_hi - ((llimb_t)(b_hi & odd) + borrow); + a_hi = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS); + + /* negate a_-b_ if it borrowed */ + a_lo ^= borrow; + a_hi ^= borrow; + limbx = a_lo + (llimb_t)(borrow & 1); + a_lo = (limb_t)limbx; + a_hi += (limb_t)(limbx >> LIMB_T_BITS) & 1; + + /* b_=a_ if a_-b_ borrowed */ + b_lo = ((t_lo ^ b_lo) & borrow) ^ b_lo; + b_hi = ((t_hi ^ b_hi) & borrow) ^ b_hi; + + /* exchange f0 and f1 if a_-b_ borrowed */ + xorm = (f0 ^ f1) & borrow; + f0 ^= xorm; + f1 ^= xorm; + + /* exchange g0 and g1 if a_-b_ borrowed */ + xorm = (g0 ^ g1) & borrow; + g0 ^= xorm; + g1 ^= xorm; + + /* subtract if a_ was odd */ + f0 -= f1 & odd; + g0 -= g1 & odd; + + f1 <<= 1; + g1 <<= 1; + a_lo >>= 1; a_lo |= a_hi << (LIMB_T_BITS-1); + a_hi >>= 1; + } + + fg->f0 = f0, fg->g0 = g0, fg->f1 = f1, fg->g1= g1; +} + +static limb_t cneg_n(limb_t ret[], const limb_t a[], limb_t neg, size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx = 0; + limb_t carry; + size_t i; + + for (carry=neg&1, i=0; i<n; i++) { + limbx = (llimb_t)(a[i] ^ neg) + carry; + ret[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + return 0 - MSB((limb_t)limbx); +} + +static limb_t add_n(limb_t ret[], const limb_t a[], limb_t b[], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t carry; + size_t i; + + for (carry=0, i=0; i<n; i++) { + limbx = a[i] + (b[i] + (llimb_t)carry); + ret[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + + return carry; +} + +static limb_t umul_n(limb_t ret[], const limb_t a[], limb_t b, size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t hi; + size_t i; + + for (hi=0, i=0; i<n; i++) { + limbx = (b * (llimb_t)a[i]) + hi; + ret[i] = (limb_t)limbx; + hi = (limb_t)(limbx >> LIMB_T_BITS); + } + + return hi; +} + +static limb_t smul_n_shift_n(limb_t ret[], const limb_t a[], limb_t *f_, + const limb_t b[], limb_t *g_, + size_t n) +{ + __builtin_assume(n != 0); + limb_t a_[n+1], b_[n+1], f, g, neg, carry, hi; + size_t i; + + /* |a|*|f_| */ + f = *f_; + neg = 0 - MSB(f); + f = (f ^ neg) - neg; /* ensure |f| is positive */ + (void)cneg_n(a_, a, neg, n); + hi = umul_n(a_, a_, f, n); + a_[n] = hi - (f & neg); + + /* |b|*|g_| */ + g = *g_; + neg = 0 - MSB(g); + g = (g ^ neg) - neg; /* ensure |g| is positive */ + (void)cneg_n(b_, b, neg, n); + hi = umul_n(b_, b_, g, n); + b_[n] = hi - (g & neg); + + /* |a|*|f_| + |b|*|g_| */ + (void)add_n(a_, a_, b_, n+1); + + /* (|a|*|f_| + |b|*|g_|) >> k */ + for (carry=a_[0], i=0; i<n; i++) { + hi = carry >> (LIMB_T_BITS-2); + carry = a_[i+1]; + ret[i] = hi | (carry << 2); + } + + /* ensure result is non-negative, fix up |f_| and |g_| accordingly */ + neg = 0 - MSB(carry); + *f_ = (*f_ ^ neg) - neg; + *g_ = (*g_ ^ neg) - neg; + (void)cneg_n(ret, ret, neg, n); + + return neg; +} + +static limb_t smul_2n(limb_t ret[], const limb_t u[], limb_t f, + const limb_t v[], limb_t g, size_t n) +{ + __builtin_assume(n != 0); + limb_t u_[n], v_[n], neg, hi; + + /* |u|*|f_| */ + neg = 0 - MSB(f); + f = (f ^ neg) - neg; /* ensure |f| is positive */ + neg = cneg_n(u_, u, neg, n); + hi = umul_n(u_, u_, f, n) - (f&neg); + + /* |v|*|g_| */ + neg = 0 - MSB(g); + g = (g ^ neg) - neg; /* ensure |g| is positive */ + neg = cneg_n(v_, v, neg, n); + hi += umul_n(v_, v_, g, n) - (g&neg); + + /* |u|*|f_| + |v|*|g_| */ + hi += add_n(ret, u_, v_, n); + + return hi; +} + +static void ct_inverse_mod_n(limb_t ret[], const limb_t inp[], + const limb_t mod[], const limb_t modx[], size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t a[n], b[n], u[2*n], v[2*n], t[2*n]; + limb_t a_[2], b_[2], sign, carry, top; + factors fg; + size_t i; + + vec_copy(a, inp, sizeof(a)); + vec_copy(b, mod, sizeof(b)); + vec_zero(u, sizeof(u)); u[0] = 1; + vec_zero(v, sizeof(v)); + + for (i=0; i<(2*n*LIMB_T_BITS)/(LIMB_T_BITS-2); i++) { + ab_approximation_n(a_, a, b_, b, n); + inner_loop_n(&fg, a_, b_, LIMB_T_BITS-2); + (void)smul_n_shift_n(t, a, &fg.f0, b, &fg.g0, n); + (void)smul_n_shift_n(b, a, &fg.f1, b, &fg.g1, n); + vec_copy(a, t, sizeof(a)); + smul_2n(t, u, fg.f0, v, fg.g0, 2*n); + smul_2n(v, u, fg.f1, v, fg.g1, 2*n); + vec_copy(u, t, sizeof(u)); + } + + inner_loop_n(&fg, a, b, (2*n*LIMB_T_BITS)%(LIMB_T_BITS-2)); + top = smul_2n(ret, u, fg.f1, v, fg.g1, 2*n); + + sign = 0 - MSB(top); /* top is 1, 0 or -1 */ + for (carry=0, i=0; i<n; i++) { + limbx = ret[n+i] + ((modx[i] & sign) + (llimb_t)carry); + ret[n+i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + top += carry; + sign = 0 - top; /* top is 1, 0 or -1 */ + top |= sign; + for (i=0; i<n; i++) + a[i] = modx[i] & top; + (void)cneg_n(a, a, 0 - MSB(sign), n); + add_n(ret+n, ret+n, a, n); +} + +#define CT_INVERSE_MOD_IMPL(bits, bits2) \ +inline void ct_inverse_mod_##bits(vec##bits2 ret, const vec##bits inp, \ + const vec##bits mod, const vec##bits modx) \ +{ ct_inverse_mod_n(ret, inp, mod, modx, NLIMBS(bits)); } + +CT_INVERSE_MOD_IMPL(256, 512) +CT_INVERSE_MOD_IMPL(384, 768) + +/* + * Copy of inner_loop_n above, but with |L| updates. + */ +static limb_t legendre_loop_n(limb_t L, factors *fg, const limb_t a_[2], + const limb_t b_[2], size_t n) +{ + __builtin_assume(n != 0); + llimb_t limbx; + limb_t f0 = 1, g0 = 0, f1 = 0, g1 = 1; + limb_t a_lo, a_hi, b_lo, b_hi, t_lo, t_hi, odd, borrow, xorm; + + a_lo = a_[0], a_hi = a_[1]; + b_lo = b_[0], b_hi = b_[1]; + + while(n--) { + odd = 0 - (a_lo&1); + + /* a_ -= b_ if a_ is odd */ + t_lo = a_lo, t_hi = a_hi; + limbx = a_lo - (llimb_t)(b_lo & odd); + a_lo = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + limbx = a_hi - ((llimb_t)(b_hi & odd) + borrow); + a_hi = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS); + + L += ((t_lo & b_lo) >> 1) & borrow; + + /* negate a_-b_ if it borrowed */ + a_lo ^= borrow; + a_hi ^= borrow; + limbx = a_lo + (llimb_t)(borrow & 1); + a_lo = (limb_t)limbx; + a_hi += (limb_t)(limbx >> LIMB_T_BITS) & 1; + + /* b_=a_ if a_-b_ borrowed */ + b_lo = ((t_lo ^ b_lo) & borrow) ^ b_lo; + b_hi = ((t_hi ^ b_hi) & borrow) ^ b_hi; + + /* exchange f0 and f1 if a_-b_ borrowed */ + xorm = (f0 ^ f1) & borrow; + f0 ^= xorm; + f1 ^= xorm; + + /* exchange g0 and g1 if a_-b_ borrowed */ + xorm = (g0 ^ g1) & borrow; + g0 ^= xorm; + g1 ^= xorm; + + /* subtract if a_ was odd */ + f0 -= f1 & odd; + g0 -= g1 & odd; + + f1 <<= 1; + g1 <<= 1; + a_lo >>= 1; a_lo |= a_hi << (LIMB_T_BITS-1); + a_hi >>= 1; + + L += (b_lo + 2) >> 2; + } + + fg->f0 = f0, fg->g0 = g0, fg->f1 = f1, fg->g1 = g1; + + return L; +} + +static bool_t ct_is_sqr_mod_n(const limb_t inp[], const limb_t mod[], size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + limb_t a[n], b[n], t[n]; + limb_t a_[2], b_[2], neg, L = 0; + factors fg; + size_t i; + + vec_copy(a, inp, sizeof(a)); + vec_copy(b, mod, sizeof(b)); + + for (i=0; i<(2*n*LIMB_T_BITS)/(LIMB_T_BITS-2); i++) { + ab_approximation_n(a_, a, b_, b, n); + L = legendre_loop_n(L, &fg, a_, b_, LIMB_T_BITS-2); + neg = smul_n_shift_n(t, a, &fg.f0, b, &fg.g0, n); + (void)smul_n_shift_n(b, a, &fg.f1, b, &fg.g1, n); + vec_copy(a, t, sizeof(a)); + L += (b[0] >> 1) & neg; + } + + L = legendre_loop_n(L, &fg, a, b, (2*n*LIMB_T_BITS)%(LIMB_T_BITS-2)); + + return (L & 1) ^ 1; +} + +#define CT_IS_SQR_MOD_IMPL(bits) \ +inline bool_t ct_is_square_mod_##bits(const vec##bits inp, \ + const vec##bits mod) \ +{ return ct_is_sqr_mod_n(inp, mod, NLIMBS(bits)); } + +CT_IS_SQR_MOD_IMPL(384) + +/* + * |div_top| points at two most significant limbs of the dividend, |d_hi| + * and |d_lo| are two most significant limbs of the divisor. If divisor + * is only one limb, it is to be passed in |d_hi| with zero in |d_lo|. + * The divisor is required to be "bitwise left-aligned," and dividend's + * top limbs to be not larger than the divisor's. The latter limitation + * can be problematic in the first iteration of multi-precision division, + * where in most general case the condition would have to be "smaller." + * The subroutine considers four limbs, two of which are "overlapping," + * hence the name... Another way to look at it is to think of the pair + * of the dividend's limbs being suffixed with a zero: + * +-------+-------+-------+ + * R | | | 0 | + * +-------+-------+-------+ + * +-------+-------+ + * D | | | + * +-------+-------+ + */ +limb_t div_3_limbs(const limb_t div_top[2], limb_t d_lo, limb_t d_hi) +{ + llimb_t Rx; + limb_t r_lo = div_top[0], r_hi = div_top[1]; + limb_t Q = 0, mask, borrow, rx; + size_t i; + + for (i = 0; i < LIMB_T_BITS; i++) { + /* "borrow, Rx = R - D" */ + Rx = (llimb_t)r_lo - d_lo; + rx = (limb_t)Rx; + borrow = (limb_t)(Rx >> LIMB_T_BITS) & 1; + Rx = r_hi - (d_hi + (llimb_t)borrow); + borrow = (limb_t)(Rx >> LIMB_T_BITS); + + /* "if (R >= D) R -= D" */ + r_lo = ((r_lo ^ rx) & borrow) ^ rx; + rx = (limb_t)Rx; + r_hi = ((r_hi ^ rx) & borrow) ^ rx; + + Q <<= 1; + Q |= ~borrow & 1; + + /* "D >>= 1" */ + d_lo >>= 1; d_lo |= d_hi << (LIMB_T_BITS - 1); + d_hi >>= 1; + } + + mask = 0 - MSB(Q); /* does it overflow? */ + + /* "borrow, Rx = R - D" */ + Rx = (llimb_t)r_lo - d_lo; + rx = (limb_t)Rx; + borrow = (limb_t)(Rx >> LIMB_T_BITS) & 1; + Rx = r_hi - (d_hi + (llimb_t)borrow); + borrow = (limb_t)(Rx >> LIMB_T_BITS) & 1; + + Q <<= 1; + Q |= borrow ^ 1; + + return (Q | mask); +} + +static limb_t quot_rem_n(limb_t *div_rem, const limb_t *divisor, + limb_t quotient, size_t n) +{ + __builtin_assume(n != 0 && n%2 == 0); + llimb_t limbx; + limb_t tmp[n+1], carry, mask, borrow; + size_t i; + + /* divisor*quotient */ + for (carry=0, i=0; i<n; i++) { + limbx = (quotient * (llimb_t)divisor[i]) + carry; + tmp[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS); + } + tmp[i] = carry; + + /* remainder = dividend - divisor*quotient */ + for (borrow=0, i=0; i<=n; i++) { + limbx = div_rem[i] - (tmp[i] + (llimb_t)borrow); + tmp[i] = (limb_t)limbx; + borrow = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + mask = 0 - borrow; + launder(mask); + + /* if quotient was off by one, add divisor to the remainder */ + for (carry=0, i=0; i<n; i++) { + limbx = tmp[i] + ((divisor[i] & mask) + (llimb_t)carry); + div_rem[i] = (limb_t)limbx; + carry = (limb_t)(limbx >> LIMB_T_BITS) & 1; + } + + return (div_rem[i] = quotient + mask); +} + +inline limb_t quot_rem_128(limb_t *div_rem, const limb_t *divisor, + limb_t quotient) +{ return quot_rem_n(div_rem, divisor, quotient, NLIMBS(128)); } + +inline limb_t quot_rem_64(limb_t *div_rem, const limb_t *divisor, + limb_t quotient) +{ return quot_rem_n(div_rem, divisor, quotient, NLIMBS(64)); } + +/* + * Unlock reference implementations in vect.c + */ +#define mul_by_8_mod_384 mul_by_8_mod_384 +#define mul_by_8_mod_384x mul_by_8_mod_384x +#define mul_by_3_mod_384x mul_by_3_mod_384x +#define mul_by_1_plus_i_mod_384x mul_by_1_plus_i_mod_384x +#define add_mod_384x add_mod_384x +#define sub_mod_384x sub_mod_384x +#define lshift_mod_384x lshift_mod_384x +#define sqr_mont_384x sqr_mont_384x + +inline void vec_prefetch(const void *ptr, size_t len) +{ (void)ptr; (void)len; } + +/* + * SHA-256 + */ +#define ROTR(x,n) ((x)>>n | (x)<<(32-n)) +#define Sigma0(x) (ROTR((x),2) ^ ROTR((x),13) ^ ROTR((x),22)) +#define Sigma1(x) (ROTR((x),6) ^ ROTR((x),11) ^ ROTR((x),25)) +#define sigma0(x) (ROTR((x),7) ^ ROTR((x),18) ^ ((x)>>3)) +#define sigma1(x) (ROTR((x),17) ^ ROTR((x),19) ^ ((x)>>10)) +#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) + +void blst_sha256_block_data_order(unsigned int *v, const void *inp, + size_t blocks) +{ + static const unsigned int K256[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + }; + unsigned int X[16], l, a, b, c, d, e, f, g, h, s0, s1, T1, T2; + const unsigned char *data = inp; + size_t round; + + a = v[0]; + b = v[1]; + c = v[2]; + d = v[3]; + e = v[4]; + f = v[5]; + g = v[6]; + h = v[7]; + + while (blocks--) { + for (round = 0; round < 16; round++) { + l = (unsigned int)data[0] << 24; + l |= (unsigned int)data[1] << 16; + l |= (unsigned int)data[2] << 8; + l |= (unsigned int)data[3]; + data += 4; + T1 = X[round] = l; + T1 += h + Sigma1(e) + Ch(e, f, g) + K256[round]; + T2 = Sigma0(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + } + + for (; round < 64; round++) { + s0 = X[(round + 1) & 0x0f]; + s0 = sigma0(s0); + s1 = X[(round + 14) & 0x0f]; + s1 = sigma1(s1); + + T1 = X[round & 0xf] += s0 + s1 + X[(round + 9) & 0xf]; + T1 += h + Sigma1(e) + Ch(e, f, g) + K256[round]; + T2 = Sigma0(a) + Maj(a, b, c); + h = g; + g = f; + f = e; + e = d + T1; + d = c; + c = b; + b = a; + a = T1 + T2; + } + + a += v[0]; v[0] = a; + b += v[1]; v[1] = b; + c += v[2]; v[2] = c; + d += v[3]; v[3] = d; + e += v[4]; v[4] = e; + f += v[5]; v[5] = f; + g += v[6]; v[6] = g; + h += v[7]; v[7] = h; + } +} +#undef ROTR +#undef Sigma0 +#undef Sigma1 +#undef sigma0 +#undef sigma1 +#undef Ch +#undef Maj + +void blst_sha256_hcopy(unsigned int dst[8], const unsigned int src[8]) +{ + size_t i; + + for (i=0; i<8; i++) + dst[i] = src[i]; +} + +void blst_sha256_emit(unsigned char md[32], const unsigned int h[8]) +{ + size_t i; + + for (i=0; i<8; i++, md+=4) { + unsigned int h_i = h[i]; + md[0] = (unsigned char)(h_i >> 24); + md[1] = (unsigned char)(h_i >> 16); + md[2] = (unsigned char)(h_i >> 8); + md[3] = (unsigned char)h_i; + } +} + +void blst_sha256_bcopy(void *dst_, const void *src_, size_t len) +{ + unsigned char *dst = dst_; + const unsigned char *src = src_; + size_t i; + + for (i=0; i<len; i++) + dst[i] = src[i]; +} diff --git a/crypto/blst_src/pairing.c b/crypto/blst_src/pairing.c new file mode 100644 index 00000000000..b256c44d68a --- /dev/null +++ b/crypto/blst_src/pairing.c @@ -0,0 +1,444 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "point.h" +#include "fields.h" + +/* + * Line evaluations from https://eprint.iacr.org/2010/354.pdf + * with a twist moving common expression to line_by_Px2. + */ +static void line_add(vec384fp6 line, POINTonE2 *T, const POINTonE2 *R, + const POINTonE2_affine *Q) +{ + vec384x Z1Z1, U2, S2, H, HH, I, J, V; +#if 1 +# define r line[1] +#else + vec384x r; +#endif + + /* + * https://hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-madd-2007-bl + * with XYZ3 being |T|, XYZ1 - |R|, XY2 - |Q|, i.e. Q is affine + */ + sqr_fp2(Z1Z1, R->Z); /* Z1Z1 = Z1^2 */ + mul_fp2(U2, Q->X, Z1Z1); /* U2 = X2*Z1Z1 */ + + mul_fp2(S2, Q->Y, R->Z); + mul_fp2(S2, S2, Z1Z1); /* S2 = Y2*Z1*Z1Z1 */ + + sub_fp2(H, U2, R->X); /* H = U2-X1 */ + + sqr_fp2(HH, H); /* HH = H^2 */ + add_fp2(I, HH, HH); + add_fp2(I, I, I); /* I = 4*HH */ + + mul_fp2(J, H, I); /* J = H*I */ + + sub_fp2(r, S2, R->Y); + add_fp2(r, r, r); /* r = 2*(S2-Y1) */ + + mul_fp2(V, R->X, I); /* V = X1*I */ + + sqr_fp2(T->X, r); + sub_fp2(T->X, T->X, J); + sub_fp2(T->X, T->X, V); + sub_fp2(T->X, T->X, V); /* X3 = r^2-J-2*V */ + + mul_fp2(J, J, R->Y); + sub_fp2(T->Y, V, T->X); + mul_fp2(T->Y, T->Y, r); + sub_fp2(T->Y, T->Y, J); + sub_fp2(T->Y, T->Y, J); /* Y3 = r*(V-X3)-2*Y1*J */ + + add_fp2(T->Z, R->Z, H); + sqr_fp2(T->Z, T->Z); + sub_fp2(T->Z, T->Z, Z1Z1); + sub_fp2(T->Z, T->Z, HH); /* Z3 = (Z1+H)^2-Z1Z1-HH */ + + /* + * line evaluation + */ + mul_fp2(I, r, Q->X); + mul_fp2(J, Q->Y, T->Z); + sub_fp2(I, I, J); + add_fp2(line[0], I, I); /* 2*(r*X2 - Y2*Z3) */ +#ifdef r +# undef r +#else + vec_copy(line[1], r, sizeof(r)); +#endif + vec_copy(line[2], T->Z, sizeof(T->Z)); +} + +static void line_dbl(vec384fp6 line, POINTonE2 *T, const POINTonE2 *Q) +{ + vec384x ZZ, A, B, C, D, E, F; + + /* + * https://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-alnr + */ + sqr_fp2(A, Q->X); /* A = X1^2 */ + sqr_fp2(B, Q->Y); /* B = Y1^2 */ + sqr_fp2(ZZ, Q->Z); /* ZZ = Z1^2 */ + sqr_fp2(C, B); /* C = B^2 */ + + add_fp2(D, Q->X, B); /* X1+B */ + sqr_fp2(D, D); /* (X1+B)^2 */ + sub_fp2(D, D, A); /* (X1+B)^2-A */ + sub_fp2(D, D, C); /* (X1+B)^2-A-C */ + add_fp2(D, D, D); /* D = 2*((X1+B)^2-A-C) */ + + mul_by_3_fp2(E, A); /* E = 3*A */ + sqr_fp2(F, E); /* F = E^2 */ + + add_fp2(line[0], E, Q->X); /* 3*A+X1 for line evaluation */ + + sub_fp2(T->X, F, D); + sub_fp2(T->X, T->X, D); /* X3 = F-2*D */ + + add_fp2(T->Z, Q->Y, Q->Z); + sqr_fp2(T->Z, T->Z); + sub_fp2(T->Z, T->Z, B); + sub_fp2(T->Z, T->Z, ZZ); /* Z3 = (Y1+Z1)^2-B-ZZ */ + + mul_by_8_fp2(C, C); /* 8*C */ + sub_fp2(T->Y, D, T->X); /* D-X3 */ + mul_fp2(T->Y, T->Y, E); /* E*(D-X3) */ + sub_fp2(T->Y, T->Y, C); /* Y3 = E*(D-X3)-8*C */ + + /* + * line evaluation + */ + sqr_fp2(line[0], line[0]); + sub_fp2(line[0], line[0], A); + sub_fp2(line[0], line[0], F); /* (3*A+X1)^2 - X1^2 - 9*A^2 */ + lshift_fp2(B, B, 2); + sub_fp2(line[0], line[0], B); /* 6*X1^3 - 4*Y1^2 */ + + mul_fp2(line[1], E, ZZ); /* 3*X1^2 * Z1^2 */ + + mul_fp2(line[2], T->Z, ZZ); /* Z3 * Z1^2 */ +} + +static void line_by_Px2(vec384fp6 line, const POINTonE1_affine *Px2) +{ + mul_fp(line[1][0], line[1][0], Px2->X); /* "b01" *= -2*P->X */ + mul_fp(line[1][1], line[1][1], Px2->X); + + mul_fp(line[2][0], line[2][0], Px2->Y); /* "b11" *= 2*P->Y */ + mul_fp(line[2][1], line[2][1], Px2->Y); +} + +#if 0 +static void add_n_dbl(vec384fp12 ret, POINTonE2 *T, const POINTonE2_affine *Q, + const POINTonE1_affine *Px2, vec384fp6 line, size_t n) +{ + line_add(line, T, T, Q); line_by_Px2(line, Px2); + mul_by_xy00z0_fp12(ret, ret, line); + while (n--) { + sqr_fp12(ret, ret); + line_dbl(line, T, T); line_by_Px2(line, Px2); + mul_by_xy00z0_fp12(ret, ret, line); + } +} + +static void miller_loop(vec384fp12 ret, const POINTonE2 *Q, const POINTonE1 *P) +{ +#define Q ((const POINTonE2_affine *)Q) + POINTonE2 T[1]; + POINTonE1_affine Px2[1]; + vec384fp6 line; /* it's not actual fp6, but 3 packed fp2, "xy00z0" */ + + /* Move common expression from line evaluation to line_by_Px2. */ + add_fp(Px2->X, P->X, P->X); + neg_fp(Px2->X, Px2->X); + add_fp(Px2->Y, P->Y, P->Y); + + vec_copy(T->X, Q->X, 2*sizeof(T->X)); + vec_copy(T->Z, BLS12_381_Rx.p2, sizeof(T->Z)); + + /* first step is ret = 1^2*line, which is replaced with ret = line */ + line_dbl(line, T, T); /* 0x2 */ + line_by_Px2(line, Px2); + vec_zero(ret, sizeof(vec384fp12)); + vec_copy(ret[0][0], line[0], 2*sizeof(vec384fp2)); + vec_copy(ret[1][1], line[2], sizeof(vec384fp2)); + add_n_dbl(ret, T, Q, Px2, line, 2); /* ..0xc */ + add_n_dbl(ret, T, Q, Px2, line, 3); /* ..0x68 */ + add_n_dbl(ret, T, Q, Px2, line, 9); /* ..0xd200 */ + add_n_dbl(ret, T, Q, Px2, line, 32); /* ..0xd20100000000 */ + add_n_dbl(ret, T, Q, Px2, line, 16); /* ..0xd201000000010000 */ + conjugate_fp12(ret); /* account for z being negative */ +#undef Q +} +#endif + +static void start_dbl_n(vec384fp12 ret, POINTonE2 T[], + const POINTonE1_affine Px2[], size_t n) +{ + size_t i; + vec384fp6 line; /* it's not actual fp6, but 3 packed fp2, "xy00z0" */ + + /* first step is ret = 1^2*line, which is replaced with ret = line */ + line_dbl(line, T+0, T+0); line_by_Px2(line, Px2+0); + vec_zero(ret, sizeof(vec384fp12)); + vec_copy(ret[0][0], line[0], 2*sizeof(vec384fp2)); + vec_copy(ret[1][1], line[2], sizeof(vec384fp2)); + + for (i = 1; i < n; i++) { + line_dbl(line, T+i, T+i); line_by_Px2(line, Px2+i); + mul_by_xy00z0_fp12(ret, ret, line); + } +} + +static void add_n_dbl_n(vec384fp12 ret, POINTonE2 T[], + const POINTonE2_affine Q[], + const POINTonE1_affine Px2[], + size_t n, size_t k) +{ + size_t i; + vec384fp6 line; /* it's not actual fp6, but 3 packed fp2, "xy00z0" */ + + for (i = 0; i < n; i++) { + line_add(line, T+i, T+i, Q+i); line_by_Px2(line, Px2+i); + mul_by_xy00z0_fp12(ret, ret, line); + } + while (k--) { + sqr_fp12(ret, ret); + for (i = 0; i < n; i++) { + line_dbl(line, T+i, T+i); line_by_Px2(line, Px2+i); + mul_by_xy00z0_fp12(ret, ret, line); + } + } +} + +static void miller_loop_n(vec384fp12 ret, const POINTonE2_affine Q[], + const POINTonE1_affine P[], size_t n) +{ +#if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 \ + || defined(__STDC_NO_VLA__) + POINTonE2 *T = alloca(n*sizeof(POINTonE2)); + POINTonE1_affine *Px2 = alloca(n*sizeof(POINTonE1_affine)); +#else + POINTonE2 T[n]; + POINTonE1_affine Px2[n]; +#endif + size_t i; + + if ((n == 1) && (vec_is_zero(&Q[0], sizeof(Q[0])) | + vec_is_zero(&P[0], sizeof(P[0]))) ) { + /* + * Special case of infinite aggregated signature, pair the additive + * group's identity with the multiplicative group's identity. + */ + vec_copy(ret, BLS12_381_Rx.p12, sizeof(vec384fp12)); + return; + } + + for (i = 0; i < n; i++) { + /* Move common expression from line evaluation to line_by_Px2. */ + add_fp(Px2[i].X, P[i].X, P[i].X); + neg_fp(Px2[i].X, Px2[i].X); + add_fp(Px2[i].Y, P[i].Y, P[i].Y); + + vec_copy(T[i].X, Q[i].X, 2*sizeof(T[i].X)); + vec_copy(T[i].Z, BLS12_381_Rx.p2, sizeof(T[i].Z)); + } + + /* first step is ret = 1^2*line, which is replaced with ret = line */ + start_dbl_n(ret, T, Px2, n); /* 0x2 */ + add_n_dbl_n(ret, T, Q, Px2, n, 2); /* ..0xc */ + add_n_dbl_n(ret, T, Q, Px2, n, 3); /* ..0x68 */ + add_n_dbl_n(ret, T, Q, Px2, n, 9); /* ..0xd200 */ + add_n_dbl_n(ret, T, Q, Px2, n, 32); /* ..0xd20100000000 */ + add_n_dbl_n(ret, T, Q, Px2, n, 16); /* ..0xd201000000010000 */ + conjugate_fp12(ret); /* account for z being negative */ +} + +static void pre_add_n_dbl(vec384fp6 lines[], POINTonE2 *T, + const POINTonE2_affine *Q, + size_t n) +{ + line_add(lines++[0], T, T, Q); + while (n--) + line_dbl(lines++[0], T, T); +} + +static void precompute_lines(vec384fp6 Qlines[68], const POINTonE2_affine *Q) +{ + POINTonE2 T[1]; + + vec_copy(T->X, Q->X, 2*sizeof(T->X)); + vec_copy(T->Z, BLS12_381_Rx.p2, sizeof(T->Z)); + + line_dbl(Qlines[0], T, T); /* 0x2 */ + pre_add_n_dbl(&Qlines[1], T, Q, 2); /* ..0xc */ + pre_add_n_dbl(&Qlines[4], T, Q, 3); /* ..0x68 */ + pre_add_n_dbl(&Qlines[8], T, Q, 9); /* ..0xd200 */ + pre_add_n_dbl(&Qlines[18], T, Q, 32); /* ..0xd20100000000 */ + pre_add_n_dbl(&Qlines[51], T, Q, 16); /* ..0xd201000000010000 */ +} + +static void post_line_by_Px2(vec384fp6 out, const vec384fp6 in, + const POINTonE1_affine *Px2) +{ + vec_copy(out[0], in[0], sizeof(out[0])); + + mul_fp(out[1][0], in[1][0], Px2->X); /* "b01" *= -2*P->X */ + mul_fp(out[1][1], in[1][1], Px2->X); + + mul_fp(out[2][0], in[2][0], Px2->Y); /* "b11" *= 2*P->Y */ + mul_fp(out[2][1], in[2][1], Px2->Y); +} + +static void post_add_n_dbl(vec384fp12 ret, const vec384fp6 lines[], + const POINTonE1_affine *Px2, size_t n) +{ + vec384fp6 line; + + post_line_by_Px2(line, lines++[0], Px2); + mul_by_xy00z0_fp12(ret, ret, line); + while (n--) { + sqr_fp12(ret, ret); + post_line_by_Px2(line, lines++[0], Px2); + mul_by_xy00z0_fp12(ret, ret, line); + } +} + +static void miller_loop_lines(vec384fp12 ret, const vec384fp6 Qlines[68], + const POINTonE1_affine *P) +{ + POINTonE1_affine Px2[1]; + vec384fp6 line; /* it's not actual fp6, but 3 packed fp2, "xy00z0" */ + + /* Move common expression from line evaluation to line_by_Px2. */ + add_fp(Px2->X, P->X, P->X); + neg_fp(Px2->X, Px2->X); + add_fp(Px2->Y, P->Y, P->Y); + + /* first step is ret = 1^2*line, which is replaced with ret = line */ + post_line_by_Px2(line, Qlines[0], Px2); /* 0x2 */ + vec_zero(ret, sizeof(vec384fp12)); + vec_copy(ret[0][0], line[0], 2*sizeof(vec384fp2)); + vec_copy(ret[1][1], line[2], sizeof(vec384fp2)); + post_add_n_dbl(ret, &Qlines[1], Px2, 2); /* ..0xc */ + post_add_n_dbl(ret, &Qlines[4], Px2, 3); /* ..0x68 */ + post_add_n_dbl(ret, &Qlines[8], Px2, 9); /* ..0xd200 */ + post_add_n_dbl(ret, &Qlines[18], Px2, 32); /* ..0xd20100000000 */ + post_add_n_dbl(ret, &Qlines[51], Px2, 16); /* ..0xd201000000010000 */ + conjugate_fp12(ret); /* account for z being negative */ +} + +#ifdef INTERNAL_TESTMODE +static void miller_loop_alt(vec384fp12 ret, const POINTonE2_affine *Q, + const POINTonE1_affine *P) +{ + vec384fp6 lines[68]; + + precompute_lines(lines, Q); + miller_loop_lines(ret, lines, P); +} +#endif + +static void mul_n_sqr(vec384fp12 ret, const vec384fp12 a, size_t n) +{ + mul_fp12(ret, ret, a); + while (n--) + cyclotomic_sqr_fp12(ret, ret); +} + +static void raise_to_z_div_by_2(vec384fp12 ret, const vec384fp12 a) +{ + cyclotomic_sqr_fp12(ret, a); /* 0x2 */ + mul_n_sqr(ret, a, 2); /* ..0xc */ + mul_n_sqr(ret, a, 3); /* ..0x68 */ + mul_n_sqr(ret, a, 9); /* ..0xd200 */ + mul_n_sqr(ret, a, 32); /* ..0xd20100000000 */ + mul_n_sqr(ret, a, 16-1); /* ..0x6900800000008000 */ + conjugate_fp12(ret); /* account for z being negative */ +} + +#define raise_to_z(a, b) (raise_to_z_div_by_2(a, b), cyclotomic_sqr_fp12(a, a)) + +/* + * Adaptation from <zkcrypto>/pairing/src/bls12_381/mod.rs + */ +static void final_exp(vec384fp12 ret, const vec384fp12 f) +{ + vec384fp12 y0, y1, y2, y3; + + vec_copy(y1, f, sizeof(y1)); + conjugate_fp12(y1); + inverse_fp12(y2, f); + mul_fp12(ret, y1, y2); + frobenius_map_fp12(y2, ret, 2); + mul_fp12(ret, ret, y2); + + cyclotomic_sqr_fp12(y0, ret); + raise_to_z(y1, y0); + raise_to_z_div_by_2(y2, y1); + vec_copy(y3, ret, sizeof(y3)); + conjugate_fp12(y3); + mul_fp12(y1, y1, y3); + conjugate_fp12(y1); + mul_fp12(y1, y1, y2); + raise_to_z(y2, y1); + raise_to_z(y3, y2); + conjugate_fp12(y1); + mul_fp12(y3, y3, y1); + conjugate_fp12(y1); + frobenius_map_fp12(y1, y1, 3); + frobenius_map_fp12(y2, y2, 2); + mul_fp12(y1, y1, y2); + raise_to_z(y2, y3); + mul_fp12(y2, y2, y0); + mul_fp12(y2, y2, ret); + mul_fp12(y1, y1, y2); + frobenius_map_fp12(y2, y3, 1); + mul_fp12(ret, y1, y2); +} + +void blst_miller_loop(vec384fp12 ret, const POINTonE2_affine *Q, + const POINTonE1_affine *P) +{ miller_loop_n(ret, Q ? Q : (const POINTonE2_affine *)&BLS12_381_G2, + P ? P : (const POINTonE1_affine *)&BLS12_381_G1, 1); +} + +void blst_final_exp(vec384fp12 ret, const vec384fp12 f) +{ final_exp(ret, f); } + +void blst_precompute_lines(vec384fp6 Qlines[68], const POINTonE2_affine *Q) +{ precompute_lines(Qlines, Q); } + +void blst_miller_loop_lines(vec384fp12 ret, const vec384fp6 Qlines[68], + const POINTonE1_affine *P) +{ miller_loop_lines(ret, Qlines, P); } + +static bool_t is_cyclotomic(const vec384fp12 f) +{ + vec384fp12 a, b; + + frobenius_map_fp12(a, f, 2); + frobenius_map_fp12(b, a, 2); + mul_fp12(b, b, f); + + return vec_is_equal(a, b, sizeof(a)); +} + +int blst_fp12_in_group(const vec384fp12 f) +{ + vec384fp12 a, b; + + if (vec_is_zero(f, sizeof(vec384fp12)) || !is_cyclotomic(f)) + return 0; + + frobenius_map_fp12(a, f, 1); + raise_to_z(b, f); + + return (int)vec_is_equal(a, b, sizeof(a)); +} diff --git a/crypto/blst_src/pentaroot-addchain.h b/crypto/blst_src/pentaroot-addchain.h new file mode 100644 index 00000000000..5bdd9ddf7f7 --- /dev/null +++ b/crypto/blst_src/pentaroot-addchain.h @@ -0,0 +1,333 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * The "magic" number is 1/5 modulo BLS12_381_r-1. Exponentiation to which + * yields 5th root of the base. + * + * Generated with 'addchain 20974350070050476191779096203274386335076221000211055129041463479975432473805' + * https://github.com/kwantam/addchain + * # Bos-Coster (win=4) : 307 (15) + * # Bos-Coster (win=10) : 307 (18) + * # Yacobi : 319 (16) + * # Bos-Coster (win=2) : 319 ( 5) + * # Bos-Coster (win=5) : 306 (19) <<< + * # Bos-Coster (win=7) : 311 (22) + * # Bos-Coster (win=9) : 313 (20) + * # Bos-Coster (win=3) : 314 ( 9) + * # Bos-Coster (win=6) : 309 (21) + * # Bos-Coster (win=8) : 309 (23) + * # Bergeron-Berstel-Brlek-Duboc : 334 ( 5) + */ + +#define PENTAROOT_MOD_BLS12_381_r(out, inp, ptype) do { \ +ptype t[19]; \ +vec_copy(t[1], inp, sizeof(ptype)); /* 0: 1 */\ +sqr(t[7], t[1]); /* 1: 2 */\ +sqr(t[0], t[7]); /* 2: 4 */\ +sqr(t[2], t[0]); /* 3: 8 */\ +mul(t[10], t[2], t[1]); /* 4: 9 */\ +mul(t[3], t[10], t[7]); /* 5: b */\ +mul(t[1], t[10], t[0]); /* 6: d */\ +mul(t[5], t[3], t[0]); /* 7: f */\ +mul(t[9], t[10], t[2]); /* 8: 11 */\ +mul(t[4], t[3], t[2]); /* 9: 13 */\ +mul(t[15], t[5], t[2]); /* 10: 17 */\ +mul(t[8], t[15], t[2]); /* 11: 1f */\ +mul(t[13], t[8], t[7]); /* 12: 21 */\ +mul(t[14], t[8], t[0]); /* 13: 23 */\ +mul(t[12], t[13], t[0]); /* 14: 25 */\ +mul(t[6], t[8], t[2]); /* 15: 27 */\ +mul(t[11], t[14], t[2]); /* 16: 2b */\ +sqr(t[0], t[15]); /* 17: 2e */\ +mul(t[18], t[6], t[2]); /* 18: 2f */\ +mul(t[2], t[11], t[2]); /* 19: 33 */\ +mul(t[16], t[2], t[7]); /* 20: 35 */\ +mul(t[7], t[0], t[3]); /* 21: 39 */\ +mul(t[17], t[0], t[5]); /* 22: 3d */\ +/* sqr(t[0], t[0]); */ /* 23: 5c */\ +/* sqr(t[0], t[0]); */ /* 24: b8 */\ +/* sqr(t[0], t[0]); */ /* 25: 170 */\ +/* sqr(t[0], t[0]); */ /* 26: 2e0 */\ +/* sqr(t[0], t[0]); */ /* 27: 5c0 */\ +/* sqr(t[0], t[0]); */ /* 28: b80 */\ +/* sqr(t[0], t[0]); */ /* 29: 1700 */\ +sqr_n_mul(t[0], t[0], 7, t[18]); /* 30: 172f */\ +/* sqr(t[0], t[0]); */ /* 31: 2e5e */\ +/* sqr(t[0], t[0]); */ /* 32: 5cbc */\ +/* sqr(t[0], t[0]); */ /* 33: b978 */\ +/* sqr(t[0], t[0]); */ /* 34: 172f0 */\ +/* sqr(t[0], t[0]); */ /* 35: 2e5e0 */\ +/* sqr(t[0], t[0]); */ /* 36: 5cbc0 */\ +sqr_n_mul(t[0], t[0], 6, t[13]); /* 37: 5cbe1 */\ +/* sqr(t[0], t[0]); */ /* 38: b97c2 */\ +/* sqr(t[0], t[0]); */ /* 39: 172f84 */\ +/* sqr(t[0], t[0]); */ /* 40: 2e5f08 */\ +/* sqr(t[0], t[0]); */ /* 41: 5cbe10 */\ +/* sqr(t[0], t[0]); */ /* 42: b97c20 */\ +/* sqr(t[0], t[0]); */ /* 43: 172f840 */\ +sqr_n_mul(t[0], t[0], 6, t[17]); /* 44: 172f87d */\ +/* sqr(t[0], t[0]); */ /* 45: 2e5f0fa */\ +/* sqr(t[0], t[0]); */ /* 46: 5cbe1f4 */\ +/* sqr(t[0], t[0]); */ /* 47: b97c3e8 */\ +/* sqr(t[0], t[0]); */ /* 48: 172f87d0 */\ +/* sqr(t[0], t[0]); */ /* 49: 2e5f0fa0 */\ +/* sqr(t[0], t[0]); */ /* 50: 5cbe1f40 */\ +sqr_n_mul(t[0], t[0], 6, t[16]); /* 51: 5cbe1f75 */\ +/* sqr(t[0], t[0]); */ /* 52: b97c3eea */\ +/* sqr(t[0], t[0]); */ /* 53: 172f87dd4 */\ +/* sqr(t[0], t[0]); */ /* 54: 2e5f0fba8 */\ +/* sqr(t[0], t[0]); */ /* 55: 5cbe1f750 */\ +/* sqr(t[0], t[0]); */ /* 56: b97c3eea0 */\ +sqr_n_mul(t[0], t[0], 5, t[15]); /* 57: b97c3eeb7 */\ +/* sqr(t[0], t[0]); */ /* 58: 172f87dd6e */\ +/* sqr(t[0], t[0]); */ /* 59: 2e5f0fbadc */\ +/* sqr(t[0], t[0]); */ /* 60: 5cbe1f75b8 */\ +/* sqr(t[0], t[0]); */ /* 61: b97c3eeb70 */\ +/* sqr(t[0], t[0]); */ /* 62: 172f87dd6e0 */\ +/* sqr(t[0], t[0]); */ /* 63: 2e5f0fbadc0 */\ +sqr_n_mul(t[0], t[0], 6, t[15]); /* 64: 2e5f0fbadd7 */\ +/* sqr(t[0], t[0]); */ /* 65: 5cbe1f75bae */\ +/* sqr(t[0], t[0]); */ /* 66: b97c3eeb75c */\ +/* sqr(t[0], t[0]); */ /* 67: 172f87dd6eb8 */\ +/* sqr(t[0], t[0]); */ /* 68: 2e5f0fbadd70 */\ +/* sqr(t[0], t[0]); */ /* 69: 5cbe1f75bae0 */\ +/* sqr(t[0], t[0]); */ /* 70: b97c3eeb75c0 */\ +/* sqr(t[0], t[0]); */ /* 71: 172f87dd6eb80 */\ +/* sqr(t[0], t[0]); */ /* 72: 2e5f0fbadd700 */\ +sqr_n_mul(t[0], t[0], 8, t[14]); /* 73: 2e5f0fbadd723 */\ +/* sqr(t[0], t[0]); */ /* 74: 5cbe1f75bae46 */\ +/* sqr(t[0], t[0]); */ /* 75: b97c3eeb75c8c */\ +/* sqr(t[0], t[0]); */ /* 76: 172f87dd6eb918 */\ +/* sqr(t[0], t[0]); */ /* 77: 2e5f0fbadd7230 */\ +/* sqr(t[0], t[0]); */ /* 78: 5cbe1f75bae460 */\ +/* sqr(t[0], t[0]); */ /* 79: b97c3eeb75c8c0 */\ +/* sqr(t[0], t[0]); */ /* 80: 172f87dd6eb9180 */\ +/* sqr(t[0], t[0]); */ /* 81: 2e5f0fbadd72300 */\ +sqr_n_mul(t[0], t[0], 8, t[13]); /* 82: 2e5f0fbadd72321 */\ +/* sqr(t[0], t[0]); */ /* 83: 5cbe1f75bae4642 */\ +/* sqr(t[0], t[0]); */ /* 84: b97c3eeb75c8c84 */\ +/* sqr(t[0], t[0]); */ /* 85: 172f87dd6eb91908 */\ +/* sqr(t[0], t[0]); */ /* 86: 2e5f0fbadd723210 */\ +/* sqr(t[0], t[0]); */ /* 87: 5cbe1f75bae46420 */\ +/* sqr(t[0], t[0]); */ /* 88: b97c3eeb75c8c840 */\ +sqr_n_mul(t[0], t[0], 6, t[2]); /* 89: b97c3eeb75c8c873 */\ +/* sqr(t[0], t[0]); */ /* 90: 172f87dd6eb9190e6 */\ +/* sqr(t[0], t[0]); */ /* 91: 2e5f0fbadd72321cc */\ +/* sqr(t[0], t[0]); */ /* 92: 5cbe1f75bae464398 */\ +/* sqr(t[0], t[0]); */ /* 93: b97c3eeb75c8c8730 */\ +/* sqr(t[0], t[0]); */ /* 94: 172f87dd6eb9190e60 */\ +/* sqr(t[0], t[0]); */ /* 95: 2e5f0fbadd72321cc0 */\ +sqr_n_mul(t[0], t[0], 6, t[13]); /* 96: 2e5f0fbadd72321ce1 */\ +/* sqr(t[0], t[0]); */ /* 97: 5cbe1f75bae46439c2 */\ +/* sqr(t[0], t[0]); */ /* 98: b97c3eeb75c8c87384 */\ +/* sqr(t[0], t[0]); */ /* 99: 172f87dd6eb9190e708 */\ +/* sqr(t[0], t[0]); */ /* 100: 2e5f0fbadd72321ce10 */\ +/* sqr(t[0], t[0]); */ /* 101: 5cbe1f75bae46439c20 */\ +/* sqr(t[0], t[0]); */ /* 102: b97c3eeb75c8c873840 */\ +/* sqr(t[0], t[0]); */ /* 103: 172f87dd6eb9190e7080 */\ +sqr_n_mul(t[0], t[0], 7, t[12]); /* 104: 172f87dd6eb9190e70a5 */\ +/* sqr(t[0], t[0]); */ /* 105: 2e5f0fbadd72321ce14a */\ +/* sqr(t[0], t[0]); */ /* 106: 5cbe1f75bae46439c294 */\ +/* sqr(t[0], t[0]); */ /* 107: b97c3eeb75c8c8738528 */\ +/* sqr(t[0], t[0]); */ /* 108: 172f87dd6eb9190e70a50 */\ +/* sqr(t[0], t[0]); */ /* 109: 2e5f0fbadd72321ce14a0 */\ +/* sqr(t[0], t[0]); */ /* 110: 5cbe1f75bae46439c2940 */\ +/* sqr(t[0], t[0]); */ /* 111: b97c3eeb75c8c87385280 */\ +/* sqr(t[0], t[0]); */ /* 112: 172f87dd6eb9190e70a500 */\ +sqr_n_mul(t[0], t[0], 8, t[11]); /* 113: 172f87dd6eb9190e70a52b */\ +/* sqr(t[0], t[0]); */ /* 114: 2e5f0fbadd72321ce14a56 */\ +/* sqr(t[0], t[0]); */ /* 115: 5cbe1f75bae46439c294ac */\ +/* sqr(t[0], t[0]); */ /* 116: b97c3eeb75c8c873852958 */\ +/* sqr(t[0], t[0]); */ /* 117: 172f87dd6eb9190e70a52b0 */\ +/* sqr(t[0], t[0]); */ /* 118: 2e5f0fbadd72321ce14a560 */\ +/* sqr(t[0], t[0]); */ /* 119: 5cbe1f75bae46439c294ac0 */\ +sqr_n_mul(t[0], t[0], 6, t[1]); /* 120: 5cbe1f75bae46439c294acd */\ +/* sqr(t[0], t[0]); */ /* 121: b97c3eeb75c8c873852959a */\ +/* sqr(t[0], t[0]); */ /* 122: 172f87dd6eb9190e70a52b34 */\ +/* sqr(t[0], t[0]); */ /* 123: 2e5f0fbadd72321ce14a5668 */\ +/* sqr(t[0], t[0]); */ /* 124: 5cbe1f75bae46439c294acd0 */\ +/* sqr(t[0], t[0]); */ /* 125: b97c3eeb75c8c873852959a0 */\ +/* sqr(t[0], t[0]); */ /* 126: 172f87dd6eb9190e70a52b340 */\ +/* sqr(t[0], t[0]); */ /* 127: 2e5f0fbadd72321ce14a56680 */\ +/* sqr(t[0], t[0]); */ /* 128: 5cbe1f75bae46439c294acd00 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 129: 5cbe1f75bae46439c294acd33 */\ +/* sqr(t[0], t[0]); */ /* 130: b97c3eeb75c8c873852959a66 */\ +/* sqr(t[0], t[0]); */ /* 131: 172f87dd6eb9190e70a52b34cc */\ +/* sqr(t[0], t[0]); */ /* 132: 2e5f0fbadd72321ce14a566998 */\ +/* sqr(t[0], t[0]); */ /* 133: 5cbe1f75bae46439c294acd330 */\ +/* sqr(t[0], t[0]); */ /* 134: b97c3eeb75c8c873852959a660 */\ +/* sqr(t[0], t[0]); */ /* 135: 172f87dd6eb9190e70a52b34cc0 */\ +sqr_n_mul(t[0], t[0], 6, t[11]); /* 136: 172f87dd6eb9190e70a52b34ceb */\ +/* sqr(t[0], t[0]); */ /* 137: 2e5f0fbadd72321ce14a56699d6 */\ +/* sqr(t[0], t[0]); */ /* 138: 5cbe1f75bae46439c294acd33ac */\ +/* sqr(t[0], t[0]); */ /* 139: b97c3eeb75c8c873852959a6758 */\ +/* sqr(t[0], t[0]); */ /* 140: 172f87dd6eb9190e70a52b34ceb0 */\ +sqr_n_mul(t[0], t[0], 4, t[10]); /* 141: 172f87dd6eb9190e70a52b34ceb9 */\ +/* sqr(t[0], t[0]); */ /* 142: 2e5f0fbadd72321ce14a56699d72 */\ +/* sqr(t[0], t[0]); */ /* 143: 5cbe1f75bae46439c294acd33ae4 */\ +/* sqr(t[0], t[0]); */ /* 144: b97c3eeb75c8c873852959a675c8 */\ +/* sqr(t[0], t[0]); */ /* 145: 172f87dd6eb9190e70a52b34ceb90 */\ +/* sqr(t[0], t[0]); */ /* 146: 2e5f0fbadd72321ce14a56699d720 */\ +sqr_n_mul(t[0], t[0], 5, t[8]); /* 147: 2e5f0fbadd72321ce14a56699d73f */\ +/* sqr(t[0], t[0]); */ /* 148: 5cbe1f75bae46439c294acd33ae7e */\ +/* sqr(t[0], t[0]); */ /* 149: b97c3eeb75c8c873852959a675cfc */\ +/* sqr(t[0], t[0]); */ /* 150: 172f87dd6eb9190e70a52b34ceb9f8 */\ +/* sqr(t[0], t[0]); */ /* 151: 2e5f0fbadd72321ce14a56699d73f0 */\ +/* sqr(t[0], t[0]); */ /* 152: 5cbe1f75bae46439c294acd33ae7e0 */\ +/* sqr(t[0], t[0]); */ /* 153: b97c3eeb75c8c873852959a675cfc0 */\ +/* sqr(t[0], t[0]); */ /* 154: 172f87dd6eb9190e70a52b34ceb9f80 */\ +/* sqr(t[0], t[0]); */ /* 155: 2e5f0fbadd72321ce14a56699d73f00 */\ +/* sqr(t[0], t[0]); */ /* 156: 5cbe1f75bae46439c294acd33ae7e00 */\ +/* sqr(t[0], t[0]); */ /* 157: b97c3eeb75c8c873852959a675cfc00 */\ +/* sqr(t[0], t[0]); */ /* 158: 172f87dd6eb9190e70a52b34ceb9f800 */\ +/* sqr(t[0], t[0]); */ /* 159: 2e5f0fbadd72321ce14a56699d73f000 */\ +/* sqr(t[0], t[0]); */ /* 160: 5cbe1f75bae46439c294acd33ae7e000 */\ +/* sqr(t[0], t[0]); */ /* 161: b97c3eeb75c8c873852959a675cfc000 */\ +/* sqr(t[0], t[0]); */ /* 162: 172f87dd6eb9190e70a52b34ceb9f8000 */\ +sqr_n_mul(t[0], t[0], 15, t[9]); /* 163: 172f87dd6eb9190e70a52b34ceb9f8011 */\ +/* sqr(t[0], t[0]); */ /* 164: 2e5f0fbadd72321ce14a56699d73f0022 */\ +/* sqr(t[0], t[0]); */ /* 165: 5cbe1f75bae46439c294acd33ae7e0044 */\ +/* sqr(t[0], t[0]); */ /* 166: b97c3eeb75c8c873852959a675cfc0088 */\ +/* sqr(t[0], t[0]); */ /* 167: 172f87dd6eb9190e70a52b34ceb9f80110 */\ +/* sqr(t[0], t[0]); */ /* 168: 2e5f0fbadd72321ce14a56699d73f00220 */\ +/* sqr(t[0], t[0]); */ /* 169: 5cbe1f75bae46439c294acd33ae7e00440 */\ +/* sqr(t[0], t[0]); */ /* 170: b97c3eeb75c8c873852959a675cfc00880 */\ +/* sqr(t[0], t[0]); */ /* 171: 172f87dd6eb9190e70a52b34ceb9f801100 */\ +sqr_n_mul(t[0], t[0], 8, t[3]); /* 172: 172f87dd6eb9190e70a52b34ceb9f80110b */\ +/* sqr(t[0], t[0]); */ /* 173: 2e5f0fbadd72321ce14a56699d73f002216 */\ +/* sqr(t[0], t[0]); */ /* 174: 5cbe1f75bae46439c294acd33ae7e00442c */\ +/* sqr(t[0], t[0]); */ /* 175: b97c3eeb75c8c873852959a675cfc008858 */\ +/* sqr(t[0], t[0]); */ /* 176: 172f87dd6eb9190e70a52b34ceb9f80110b0 */\ +/* sqr(t[0], t[0]); */ /* 177: 2e5f0fbadd72321ce14a56699d73f0022160 */\ +sqr_n_mul(t[0], t[0], 5, t[8]); /* 178: 2e5f0fbadd72321ce14a56699d73f002217f */\ +/* sqr(t[0], t[0]); */ /* 179: 5cbe1f75bae46439c294acd33ae7e00442fe */\ +/* sqr(t[0], t[0]); */ /* 180: b97c3eeb75c8c873852959a675cfc00885fc */\ +/* sqr(t[0], t[0]); */ /* 181: 172f87dd6eb9190e70a52b34ceb9f80110bf8 */\ +/* sqr(t[0], t[0]); */ /* 182: 2e5f0fbadd72321ce14a56699d73f002217f0 */\ +/* sqr(t[0], t[0]); */ /* 183: 5cbe1f75bae46439c294acd33ae7e00442fe0 */\ +/* sqr(t[0], t[0]); */ /* 184: b97c3eeb75c8c873852959a675cfc00885fc0 */\ +/* sqr(t[0], t[0]); */ /* 185: 172f87dd6eb9190e70a52b34ceb9f80110bf80 */\ +/* sqr(t[0], t[0]); */ /* 186: 2e5f0fbadd72321ce14a56699d73f002217f00 */\ +/* sqr(t[0], t[0]); */ /* 187: 5cbe1f75bae46439c294acd33ae7e00442fe00 */\ +/* sqr(t[0], t[0]); */ /* 188: b97c3eeb75c8c873852959a675cfc00885fc00 */\ +sqr_n_mul(t[0], t[0], 10, t[7]); /* 189: b97c3eeb75c8c873852959a675cfc00885fc39 */\ +/* sqr(t[0], t[0]); */ /* 190: 172f87dd6eb9190e70a52b34ceb9f80110bf872 */\ +/* sqr(t[0], t[0]); */ /* 191: 2e5f0fbadd72321ce14a56699d73f002217f0e4 */\ +/* sqr(t[0], t[0]); */ /* 192: 5cbe1f75bae46439c294acd33ae7e00442fe1c8 */\ +/* sqr(t[0], t[0]); */ /* 193: b97c3eeb75c8c873852959a675cfc00885fc390 */\ +/* sqr(t[0], t[0]); */ /* 194: 172f87dd6eb9190e70a52b34ceb9f80110bf8720 */\ +/* sqr(t[0], t[0]); */ /* 195: 2e5f0fbadd72321ce14a56699d73f002217f0e40 */\ +sqr_n_mul(t[0], t[0], 6, t[6]); /* 196: 2e5f0fbadd72321ce14a56699d73f002217f0e67 */\ +/* sqr(t[0], t[0]); */ /* 197: 5cbe1f75bae46439c294acd33ae7e00442fe1cce */\ +/* sqr(t[0], t[0]); */ /* 198: b97c3eeb75c8c873852959a675cfc00885fc399c */\ +/* sqr(t[0], t[0]); */ /* 199: 172f87dd6eb9190e70a52b34ceb9f80110bf87338 */\ +/* sqr(t[0], t[0]); */ /* 200: 2e5f0fbadd72321ce14a56699d73f002217f0e670 */\ +/* sqr(t[0], t[0]); */ /* 201: 5cbe1f75bae46439c294acd33ae7e00442fe1cce0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 202: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3 */\ +/* sqr(t[0], t[0]); */ /* 203: b97c3eeb75c8c873852959a675cfc00885fc399e6 */\ +/* sqr(t[0], t[0]); */ /* 204: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cc */\ +/* sqr(t[0], t[0]); */ /* 205: 2e5f0fbadd72321ce14a56699d73f002217f0e6798 */\ +/* sqr(t[0], t[0]); */ /* 206: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf30 */\ +/* sqr(t[0], t[0]); */ /* 207: b97c3eeb75c8c873852959a675cfc00885fc399e60 */\ +/* sqr(t[0], t[0]); */ /* 208: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cc0 */\ +/* sqr(t[0], t[0]); */ /* 209: 2e5f0fbadd72321ce14a56699d73f002217f0e67980 */\ +/* sqr(t[0], t[0]); */ /* 210: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 211: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf333 */\ +/* sqr(t[0], t[0]); */ /* 212: b97c3eeb75c8c873852959a675cfc00885fc399e666 */\ +/* sqr(t[0], t[0]); */ /* 213: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc */\ +/* sqr(t[0], t[0]); */ /* 214: 2e5f0fbadd72321ce14a56699d73f002217f0e679998 */\ +/* sqr(t[0], t[0]); */ /* 215: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3330 */\ +/* sqr(t[0], t[0]); */ /* 216: b97c3eeb75c8c873852959a675cfc00885fc399e6660 */\ +/* sqr(t[0], t[0]); */ /* 217: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc0 */\ +/* sqr(t[0], t[0]); */ /* 218: 2e5f0fbadd72321ce14a56699d73f002217f0e6799980 */\ +sqr_n_mul(t[0], t[0], 7, t[5]); /* 219: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f */\ +/* sqr(t[0], t[0]); */ /* 220: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e */\ +/* sqr(t[0], t[0]); */ /* 221: b97c3eeb75c8c873852959a675cfc00885fc399e6663c */\ +/* sqr(t[0], t[0]); */ /* 222: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78 */\ +/* sqr(t[0], t[0]); */ /* 223: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f0 */\ +/* sqr(t[0], t[0]); */ /* 224: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e0 */\ +/* sqr(t[0], t[0]); */ /* 225: b97c3eeb75c8c873852959a675cfc00885fc399e6663c0 */\ +/* sqr(t[0], t[0]); */ /* 226: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc780 */\ +/* sqr(t[0], t[0]); */ /* 227: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f00 */\ +/* sqr(t[0], t[0]); */ /* 228: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e00 */\ +sqr_n_mul(t[0], t[0], 9, t[2]); /* 229: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33 */\ +/* sqr(t[0], t[0]); */ /* 230: b97c3eeb75c8c873852959a675cfc00885fc399e6663c66 */\ +/* sqr(t[0], t[0]); */ /* 231: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc */\ +/* sqr(t[0], t[0]); */ /* 232: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f198 */\ +/* sqr(t[0], t[0]); */ /* 233: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e330 */\ +/* sqr(t[0], t[0]); */ /* 234: b97c3eeb75c8c873852959a675cfc00885fc399e6663c660 */\ +/* sqr(t[0], t[0]); */ /* 235: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc0 */\ +/* sqr(t[0], t[0]); */ /* 236: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f1980 */\ +sqr_n_mul(t[0], t[0], 7, t[4]); /* 237: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f1993 */\ +/* sqr(t[0], t[0]); */ /* 238: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e3326 */\ +/* sqr(t[0], t[0]); */ /* 239: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664c */\ +/* sqr(t[0], t[0]); */ /* 240: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc98 */\ +/* sqr(t[0], t[0]); */ /* 241: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19930 */\ +/* sqr(t[0], t[0]); */ /* 242: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33260 */\ +/* sqr(t[0], t[0]); */ /* 243: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664c0 */\ +/* sqr(t[0], t[0]); */ /* 244: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc980 */\ +/* sqr(t[0], t[0]); */ /* 245: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f199300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 246: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f199333 */\ +/* sqr(t[0], t[0]); */ /* 247: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e332666 */\ +/* sqr(t[0], t[0]); */ /* 248: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccc */\ +/* sqr(t[0], t[0]); */ /* 249: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc9998 */\ +/* sqr(t[0], t[0]); */ /* 250: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f1993330 */\ +/* sqr(t[0], t[0]); */ /* 251: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e3326660 */\ +/* sqr(t[0], t[0]); */ /* 252: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccc0 */\ +/* sqr(t[0], t[0]); */ /* 253: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc99980 */\ +/* sqr(t[0], t[0]); */ /* 254: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 255: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333 */\ +/* sqr(t[0], t[0]); */ /* 256: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33266666 */\ +/* sqr(t[0], t[0]); */ /* 257: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccccc */\ +/* sqr(t[0], t[0]); */ /* 258: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc999998 */\ +/* sqr(t[0], t[0]); */ /* 259: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f199333330 */\ +/* sqr(t[0], t[0]); */ /* 260: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e332666660 */\ +/* sqr(t[0], t[0]); */ /* 261: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccccc0 */\ +/* sqr(t[0], t[0]); */ /* 262: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc9999980 */\ +/* sqr(t[0], t[0]); */ /* 263: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f1993333300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 264: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f1993333333 */\ +/* sqr(t[0], t[0]); */ /* 265: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e3326666666 */\ +/* sqr(t[0], t[0]); */ /* 266: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccccccc */\ +/* sqr(t[0], t[0]); */ /* 267: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc99999998 */\ +/* sqr(t[0], t[0]); */ /* 268: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333330 */\ +/* sqr(t[0], t[0]); */ /* 269: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33266666660 */\ +/* sqr(t[0], t[0]); */ /* 270: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664ccccccc0 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 271: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb */\ +/* sqr(t[0], t[0]); */ /* 272: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc999999996 */\ +/* sqr(t[0], t[0]); */ /* 273: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332c */\ +/* sqr(t[0], t[0]); */ /* 274: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e332666666658 */\ +/* sqr(t[0], t[0]); */ /* 275: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb0 */\ +/* sqr(t[0], t[0]); */ /* 276: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc9999999960 */\ +/* sqr(t[0], t[0]); */ /* 277: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332c0 */\ +/* sqr(t[0], t[0]); */ /* 278: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e3326666666580 */\ +/* sqr(t[0], t[0]); */ /* 279: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb00 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 280: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb33 */\ +/* sqr(t[0], t[0]); */ /* 281: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc99999999666 */\ +/* sqr(t[0], t[0]); */ /* 282: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccc */\ +/* sqr(t[0], t[0]); */ /* 283: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33266666665998 */\ +/* sqr(t[0], t[0]); */ /* 284: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb330 */\ +/* sqr(t[0], t[0]); */ /* 285: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc999999996660 */\ +/* sqr(t[0], t[0]); */ /* 286: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccc0 */\ +/* sqr(t[0], t[0]); */ /* 287: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e332666666659980 */\ +/* sqr(t[0], t[0]); */ /* 288: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb3300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 289: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb3333 */\ +/* sqr(t[0], t[0]); */ /* 290: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc9999999966666 */\ +/* sqr(t[0], t[0]); */ /* 291: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccccc */\ +/* sqr(t[0], t[0]); */ /* 292: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e3326666666599998 */\ +/* sqr(t[0], t[0]); */ /* 293: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb33330 */\ +/* sqr(t[0], t[0]); */ /* 294: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc99999999666660 */\ +/* sqr(t[0], t[0]); */ /* 295: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccccc0 */\ +/* sqr(t[0], t[0]); */ /* 296: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e33266666665999980 */\ +/* sqr(t[0], t[0]); */ /* 297: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb333300 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 298: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb333333 */\ +/* sqr(t[0], t[0]); */ /* 299: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc999999996666666 */\ +/* sqr(t[0], t[0]); */ /* 300: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccccccc */\ +/* sqr(t[0], t[0]); */ /* 301: 5cbe1f75bae46439c294acd33ae7e00442fe1ccf3331e332666666659999998 */\ +/* sqr(t[0], t[0]); */ /* 302: b97c3eeb75c8c873852959a675cfc00885fc399e6663c664cccccccb3333330 */\ +/* sqr(t[0], t[0]); */ /* 303: 172f87dd6eb9190e70a52b34ceb9f80110bf8733cccc78cc9999999966666660 */\ +/* sqr(t[0], t[0]); */ /* 304: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332ccccccc0 */\ +sqr_n_mul(out, t[0], 6, t[1]); /* 305: 2e5f0fbadd72321ce14a56699d73f002217f0e679998f19933333332cccccccd */\ +} while(0) diff --git a/crypto/blst_src/pentaroot.c b/crypto/blst_src/pentaroot.c new file mode 100644 index 00000000000..fd028113f3d --- /dev/null +++ b/crypto/blst_src/pentaroot.c @@ -0,0 +1,76 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" + +static inline void mul_fr(vec384 ret, const vec384 a, const vec384 b) +{ mul_mont_sparse_256(ret, a, b, BLS12_381_r, r0); } + +static inline void sqr_fr(vec384 ret, const vec384 a) +{ sqr_mont_sparse_256(ret, a, BLS12_381_r, r0); } + +#ifdef __OPTIMIZE_SIZE__ +void blst_fr_pentaroot(vec256 out, const vec256 inp) +{ + static const byte pow[] = { + TO_BYTES(0x33333332cccccccd), TO_BYTES(0x217f0e679998f199), + TO_BYTES(0xe14a56699d73f002), TO_BYTES(0x2e5f0fbadd72321c) + }; + size_t pow_bits = 254; + vec256 ret; + + vec_copy(ret, inp, sizeof(ret)); /* ret = inp^1 */ + --pow_bits; /* most significant bit is set, skip over */ + while (pow_bits--) { + sqr_fr(ret, ret); + if (is_bit_set(pow, pow_bits)) + mul_fr(ret, ret, inp); + } + vec_copy(out, ret, sizeof(ret)); /* out = ret */ +} +#else +# if 0 +/* + * "255"-bit variant omits full reductions at the ends of squarings, + * not implemented yet[?]. + */ +static inline void sqr_n_mul_fr(vec256 out, const vec256 a, size_t count, + const vec256 b) +{ sqr_n_mul_mont_255(out, a, count, BLS12_381_r, r0, b); } +# else +static void sqr_n_mul_fr(vec256 out, const vec256 a, size_t count, + const vec256 b) +{ + do { + sqr_fr(out, a); + a = out; + } while (--count); + mul_fr(out, out, b); +} +# endif + +# define sqr(ret,a) sqr_fr(ret,a) +# define mul(ret,a,b) mul_fr(ret,a,b) +# define sqr_n_mul(ret,a,n,b) sqr_n_mul_fr(ret,a,n,b) + +# include "pentaroot-addchain.h" +void blst_fr_pentaroot(vec256 out, const vec256 inp) +{ PENTAROOT_MOD_BLS12_381_r(out, inp, vec256); } +# undef PENTAROOT_MOD_BLS12_381_r + +# undef sqr_n_mul +# undef sqr +# undef mul +#endif + +void blst_fr_pentapow(vec256 out, const vec256 inp) +{ + vec256 tmp; + + sqr_fr(tmp, inp); + sqr_fr(tmp, tmp); + mul_fr(out, tmp, inp); +} diff --git a/crypto/blst_src/point.h b/crypto/blst_src/point.h new file mode 100644 index 00000000000..0aa7379671f --- /dev/null +++ b/crypto/blst_src/point.h @@ -0,0 +1,62 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_POINT_H__ +#define __BLS12_381_ASM_POINT_H__ + +#include "vect.h" +#include "bytes.h" + +#define DECLARE_POINT(ptype, bits) \ +typedef struct { vec##bits X,Y,Z; } ptype; \ +typedef struct { vec##bits X,Y; } ptype##_affine; \ +\ +static void ptype##_dadd(ptype *out, const ptype *p1, const ptype *p2, \ + const vec##bits a4); \ +static void ptype##_dadd_affine(ptype *out, const ptype *p1, \ + const ptype##_affine *p2); \ +static void ptype##_add(ptype *out, const ptype *p1, const ptype *p2); \ +static void ptype##_add_affine(ptype *out, const ptype *p1, \ + const ptype##_affine *p2); \ +static void ptype##_double(ptype *out, const ptype *p1); \ +static void ptype##_mult_w5(ptype *out, const ptype *point, \ + const byte *scalar, size_t nbits); \ +static void ptype##_cneg(ptype *p, limb_t cbit); \ +static void ptype##_to_affine(ptype##_affine *out, const ptype *in); \ +static void ptype##_from_Jacobian(ptype *out, const ptype *in); \ +\ +static inline void ptype##_cswap(ptype *restrict a, \ + ptype *restrict b, bool_t cbit) { \ + vec_cswap(a, b, sizeof(ptype), cbit); \ +} \ +static inline void ptype##_ccopy(ptype *restrict a, \ + const ptype *restrict b, bool_t cbit) {\ + vec_select(a, b, a, sizeof(ptype), cbit); \ +} + +#define DECLARE_PRIVATE_POINTXZ(ptype, bits) \ +typedef struct { vec##bits X,Z; } ptype##xz; \ +\ +static void ptype##xz_ladder_pre(ptype##xz *out, const ptype *in); \ +static void ptype##xz_ladder_step(ptype##xz *r, ptype##xz *s, \ + const ptype##xz *p); \ +static void ptype##xz_ladder_post(ptype *ret, \ + const ptype##xz *r, const ptype##xz *s, \ + const ptype##xz *p, const vec##bits Y1);\ +\ +static inline void ptype##xz_cswap(ptype##xz *restrict a, \ + ptype##xz *restrict b, bool_t cbit) {\ + vec_cswap(a, b, sizeof(ptype##xz), cbit); \ +} + +DECLARE_POINT(POINTonE1, 384) + +DECLARE_POINT(POINTonE2, 384x) + +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +#endif diff --git a/crypto/blst_src/rb_tree.c b/crypto/blst_src/rb_tree.c new file mode 100644 index 00000000000..207becdad18 --- /dev/null +++ b/crypto/blst_src/rb_tree.c @@ -0,0 +1,145 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include <stddef.h> + +/* + * Red-black tree tailored for uniqueness test. Amount of messages to be + * checked is known prior context initialization, implementation is + * insert-only, failure is returned if message is already in the tree. + */ + +struct node { + struct node *leafs[2]; + const void *data; + size_t len_n_colour; /* len<<1 | colour */ +}; + +struct rb_tree { + struct node *root; + size_t n_nodes; + struct node nodes[1]; +}; + +static long bytes_compare(const unsigned char *ptr0, size_t len0, + const unsigned char *ptr1, size_t len1) +{ + size_t i, len = len0<len1 ? len0 : len1; + long a, b; + + for (i=0; i<len; i++) { + if ((a = ptr0[i]) != (b = ptr1[i])) + return a - b; + } + + return (long)len0 - (long)len1; +} + +#define PAINT_BLACK(p) ((p)->len_n_colour &= ~(size_t)1) +#define PAINT_RED(p) ((p)->len_n_colour |= 1) +#define IS_RED(p) ((p)->len_n_colour & 1) + +static int rb_tree_insert(struct rb_tree *tree, const void *data, size_t len) +{ + struct node *nodes[8*sizeof(void *)]; /* visited nodes */ + unsigned char dirs[8*sizeof(void *)]; /* taken directions */ + size_t k = 0; /* walked distance */ + struct node *p, *y, *z; + + for (p = tree->root; p != NULL; k++) { + long cmp = bytes_compare(data, len, p->data, p->len_n_colour>>1); + + if (cmp == 0) + return 0; /* already in tree, no insertion */ + + /* record the step */ + nodes[k] = p; + p = p->leafs[(dirs[k] = cmp>0)]; + } + + /* allocate new node */ + z = &tree->nodes[tree->n_nodes++]; + z->leafs[0] = z->leafs[1] = NULL; + z->data = data; + z->len_n_colour = len<<1; + PAINT_RED(z); + + /* graft |z| */ + if (k > 0) + nodes[k-1]->leafs[dirs[k-1]] = z; + else + tree->root = z; + + /* re-balance |tree| */ + while (k >= 2 && IS_RED(y = nodes[k-1])) { + size_t ydir = dirs[k-2]; + struct node *x = nodes[k-2], /* |z|'s grandparent */ + *s = x->leafs[ydir^1]; /* |z|'s uncle */ + + if (s != NULL && IS_RED(s)) { + PAINT_RED(x); + PAINT_BLACK(y); + PAINT_BLACK(s); + k -= 2; + } else { + if (dirs[k-1] != ydir) { + /* | | + * x x + * / \ \ + * y s -> z s + * \ / + * z y + * / \ + * ? ? + */ + struct node *t = y; + y = y->leafs[ydir^1]; + t->leafs[ydir^1] = y->leafs[ydir]; + y->leafs[ydir] = t; + } + + /* | | + * x y + * \ / \ + * y s -> z x + * / \ / \ + * z ? ? s + */ + x->leafs[ydir] = y->leafs[ydir^1]; + y->leafs[ydir^1] = x; + + PAINT_RED(x); + PAINT_BLACK(y); + + if (k > 2) + nodes[k-3]->leafs[dirs[k-3]] = y; + else + tree->root = y; + + break; + } + } + + PAINT_BLACK(tree->root); + + return 1; +} + +#undef IS_RED +#undef PAINT_RED +#undef PAINT_BLACK + +size_t blst_uniq_sizeof(size_t n_nodes) +{ return sizeof(struct rb_tree) + sizeof(struct node)*(n_nodes-1); } + +void blst_uniq_init(struct rb_tree *tree) +{ + tree->root = NULL; + tree->n_nodes = 0; +} + +int blst_uniq_test(struct rb_tree *tree, const void *data, size_t len) +{ return (int)rb_tree_insert(tree, data, len); } diff --git a/crypto/blst_src/recip-addchain.h b/crypto/blst_src/recip-addchain.h new file mode 100644 index 00000000000..e4e436a3f09 --- /dev/null +++ b/crypto/blst_src/recip-addchain.h @@ -0,0 +1,489 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * The "magic" number is BLS12_381_P-2. Exponentiation to which yields + * reciprocal to input base. + * + * Generated with 'addchain 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559785' + * https://github.com/kwantam/addchain + * + * # Bos-Coster (win=4) : 461 (16) <<< + * # Bos-Coster (win=3) : 464 ( 9) + * # Bos-Coster (win=8) : 469 (35) + * # Bos-Coster (win=5) : 463 (28) + * # Bos-Coster (win=9) : 467 (32) + * # Bos-Coster (win=7) : 462 (27) + * # Yacobi : 481 (31) + * # Bos-Coster (win=10) : 475 (30) + * # Bos-Coster (win=6) : 463 (32) + * # Bos-Coster (win=2) : 489 ( 5) + * # Bergeron-Berstel-Brlek-Duboc : 498 ( 5) + */ + +#define RECIPROCAL_MOD_BLS12_381_P(out, inp, ptype) do { \ +ptype t[16]; \ +vec_copy(t[1], inp, sizeof(ptype)); /* 0: 1 */\ +sqr(t[0], t[1]); /* 1: 2 */\ +mul(t[9], t[0], t[1]); /* 2: 3 */\ +sqr(t[5], t[0]); /* 3: 4 */\ +mul(t[2], t[9], t[0]); /* 4: 5 */\ +mul(t[7], t[5], t[9]); /* 5: 7 */\ +mul(t[10], t[2], t[5]); /* 6: 9 */\ +mul(t[13], t[7], t[5]); /* 7: b */\ +mul(t[4], t[10], t[5]); /* 8: d */\ +mul(t[8], t[13], t[5]); /* 9: f */\ +mul(t[15], t[4], t[5]); /* 10: 11 */\ +mul(t[11], t[8], t[5]); /* 11: 13 */\ +mul(t[3], t[15], t[5]); /* 12: 15 */\ +mul(t[12], t[11], t[5]); /* 13: 17 */\ +sqr(t[0], t[4]); /* 14: 1a */\ +mul(t[14], t[12], t[5]); /* 15: 1b */\ +mul(t[6], t[0], t[9]); /* 16: 1d */\ +mul(t[5], t[0], t[2]); /* 17: 1f */\ +/* sqr(t[0], t[0]); */ /* 18: 34 */\ +/* sqr(t[0], t[0]); */ /* 19: 68 */\ +/* sqr(t[0], t[0]); */ /* 20: d0 */\ +/* sqr(t[0], t[0]); */ /* 21: 1a0 */\ +/* sqr(t[0], t[0]); */ /* 22: 340 */\ +/* sqr(t[0], t[0]); */ /* 23: 680 */\ +/* sqr(t[0], t[0]); */ /* 24: d00 */\ +/* sqr(t[0], t[0]); */ /* 25: 1a00 */\ +/* sqr(t[0], t[0]); */ /* 26: 3400 */\ +/* sqr(t[0], t[0]); */ /* 27: 6800 */\ +/* sqr(t[0], t[0]); */ /* 28: d000 */\ +/* sqr(t[0], t[0]); */ /* 29: 1a000 */\ +sqr_n_mul(t[0], t[0], 12, t[15]); /* 30: 1a011 */\ +/* sqr(t[0], t[0]); */ /* 31: 34022 */\ +/* sqr(t[0], t[0]); */ /* 32: 68044 */\ +/* sqr(t[0], t[0]); */ /* 33: d0088 */\ +/* sqr(t[0], t[0]); */ /* 34: 1a0110 */\ +/* sqr(t[0], t[0]); */ /* 35: 340220 */\ +/* sqr(t[0], t[0]); */ /* 36: 680440 */\ +/* sqr(t[0], t[0]); */ /* 37: d00880 */\ +sqr_n_mul(t[0], t[0], 7, t[8]); /* 38: d0088f */\ +/* sqr(t[0], t[0]); */ /* 39: 1a0111e */\ +/* sqr(t[0], t[0]); */ /* 40: 340223c */\ +/* sqr(t[0], t[0]); */ /* 41: 6804478 */\ +/* sqr(t[0], t[0]); */ /* 42: d0088f0 */\ +sqr_n_mul(t[0], t[0], 4, t[2]); /* 43: d0088f5 */\ +/* sqr(t[0], t[0]); */ /* 44: 1a0111ea */\ +/* sqr(t[0], t[0]); */ /* 45: 340223d4 */\ +/* sqr(t[0], t[0]); */ /* 46: 680447a8 */\ +/* sqr(t[0], t[0]); */ /* 47: d0088f50 */\ +/* sqr(t[0], t[0]); */ /* 48: 1a0111ea0 */\ +/* sqr(t[0], t[0]); */ /* 49: 340223d40 */\ +sqr_n_mul(t[0], t[0], 6, t[7]); /* 50: 340223d47 */\ +/* sqr(t[0], t[0]); */ /* 51: 680447a8e */\ +/* sqr(t[0], t[0]); */ /* 52: d0088f51c */\ +/* sqr(t[0], t[0]); */ /* 53: 1a0111ea38 */\ +/* sqr(t[0], t[0]); */ /* 54: 340223d470 */\ +/* sqr(t[0], t[0]); */ /* 55: 680447a8e0 */\ +/* sqr(t[0], t[0]); */ /* 56: d0088f51c0 */\ +/* sqr(t[0], t[0]); */ /* 57: 1a0111ea380 */\ +sqr_n_mul(t[0], t[0], 7, t[12]); /* 58: 1a0111ea397 */\ +/* sqr(t[0], t[0]); */ /* 59: 340223d472e */\ +/* sqr(t[0], t[0]); */ /* 60: 680447a8e5c */\ +/* sqr(t[0], t[0]); */ /* 61: d0088f51cb8 */\ +/* sqr(t[0], t[0]); */ /* 62: 1a0111ea3970 */\ +/* sqr(t[0], t[0]); */ /* 63: 340223d472e0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 64: 340223d472ff */\ +/* sqr(t[0], t[0]); */ /* 65: 680447a8e5fe */\ +/* sqr(t[0], t[0]); */ /* 66: d0088f51cbfc */\ +sqr_n_mul(t[0], t[0], 2, t[9]); /* 67: d0088f51cbff */\ +/* sqr(t[0], t[0]); */ /* 68: 1a0111ea397fe */\ +/* sqr(t[0], t[0]); */ /* 69: 340223d472ffc */\ +/* sqr(t[0], t[0]); */ /* 70: 680447a8e5ff8 */\ +/* sqr(t[0], t[0]); */ /* 71: d0088f51cbff0 */\ +/* sqr(t[0], t[0]); */ /* 72: 1a0111ea397fe0 */\ +/* sqr(t[0], t[0]); */ /* 73: 340223d472ffc0 */\ +sqr_n_mul(t[0], t[0], 6, t[4]); /* 74: 340223d472ffcd */\ +/* sqr(t[0], t[0]); */ /* 75: 680447a8e5ff9a */\ +/* sqr(t[0], t[0]); */ /* 76: d0088f51cbff34 */\ +/* sqr(t[0], t[0]); */ /* 77: 1a0111ea397fe68 */\ +/* sqr(t[0], t[0]); */ /* 78: 340223d472ffcd0 */\ +/* sqr(t[0], t[0]); */ /* 79: 680447a8e5ff9a0 */\ +/* sqr(t[0], t[0]); */ /* 80: d0088f51cbff340 */\ +sqr_n_mul(t[0], t[0], 6, t[4]); /* 81: d0088f51cbff34d */\ +/* sqr(t[0], t[0]); */ /* 82: 1a0111ea397fe69a */\ +/* sqr(t[0], t[0]); */ /* 83: 340223d472ffcd34 */\ +/* sqr(t[0], t[0]); */ /* 84: 680447a8e5ff9a68 */\ +/* sqr(t[0], t[0]); */ /* 85: d0088f51cbff34d0 */\ +/* sqr(t[0], t[0]); */ /* 86: 1a0111ea397fe69a0 */\ +/* sqr(t[0], t[0]); */ /* 87: 340223d472ffcd340 */\ +sqr_n_mul(t[0], t[0], 6, t[10]); /* 88: 340223d472ffcd349 */\ +/* sqr(t[0], t[0]); */ /* 89: 680447a8e5ff9a692 */\ +/* sqr(t[0], t[0]); */ /* 90: d0088f51cbff34d24 */\ +/* sqr(t[0], t[0]); */ /* 91: 1a0111ea397fe69a48 */\ +sqr_n_mul(t[0], t[0], 3, t[9]); /* 92: 1a0111ea397fe69a4b */\ +/* sqr(t[0], t[0]); */ /* 93: 340223d472ffcd3496 */\ +/* sqr(t[0], t[0]); */ /* 94: 680447a8e5ff9a692c */\ +/* sqr(t[0], t[0]); */ /* 95: d0088f51cbff34d258 */\ +/* sqr(t[0], t[0]); */ /* 96: 1a0111ea397fe69a4b0 */\ +/* sqr(t[0], t[0]); */ /* 97: 340223d472ffcd34960 */\ +/* sqr(t[0], t[0]); */ /* 98: 680447a8e5ff9a692c0 */\ +/* sqr(t[0], t[0]); */ /* 99: d0088f51cbff34d2580 */\ +sqr_n_mul(t[0], t[0], 7, t[4]); /* 100: d0088f51cbff34d258d */\ +/* sqr(t[0], t[0]); */ /* 101: 1a0111ea397fe69a4b1a */\ +/* sqr(t[0], t[0]); */ /* 102: 340223d472ffcd349634 */\ +/* sqr(t[0], t[0]); */ /* 103: 680447a8e5ff9a692c68 */\ +/* sqr(t[0], t[0]); */ /* 104: d0088f51cbff34d258d0 */\ +sqr_n_mul(t[0], t[0], 4, t[4]); /* 105: d0088f51cbff34d258dd */\ +/* sqr(t[0], t[0]); */ /* 106: 1a0111ea397fe69a4b1ba */\ +/* sqr(t[0], t[0]); */ /* 107: 340223d472ffcd3496374 */\ +/* sqr(t[0], t[0]); */ /* 108: 680447a8e5ff9a692c6e8 */\ +/* sqr(t[0], t[0]); */ /* 109: d0088f51cbff34d258dd0 */\ +/* sqr(t[0], t[0]); */ /* 110: 1a0111ea397fe69a4b1ba0 */\ +/* sqr(t[0], t[0]); */ /* 111: 340223d472ffcd34963740 */\ +sqr_n_mul(t[0], t[0], 6, t[8]); /* 112: 340223d472ffcd3496374f */\ +/* sqr(t[0], t[0]); */ /* 113: 680447a8e5ff9a692c6e9e */\ +/* sqr(t[0], t[0]); */ /* 114: d0088f51cbff34d258dd3c */\ +/* sqr(t[0], t[0]); */ /* 115: 1a0111ea397fe69a4b1ba78 */\ +/* sqr(t[0], t[0]); */ /* 116: 340223d472ffcd3496374f0 */\ +/* sqr(t[0], t[0]); */ /* 117: 680447a8e5ff9a692c6e9e0 */\ +/* sqr(t[0], t[0]); */ /* 118: d0088f51cbff34d258dd3c0 */\ +sqr_n_mul(t[0], t[0], 6, t[14]); /* 119: d0088f51cbff34d258dd3db */\ +/* sqr(t[0], t[0]); */ /* 120: 1a0111ea397fe69a4b1ba7b6 */\ +/* sqr(t[0], t[0]); */ /* 121: 340223d472ffcd3496374f6c */\ +/* sqr(t[0], t[0]); */ /* 122: 680447a8e5ff9a692c6e9ed8 */\ +sqr_n_mul(t[0], t[0], 3, t[1]); /* 123: 680447a8e5ff9a692c6e9ed9 */\ +/* sqr(t[0], t[0]); */ /* 124: d0088f51cbff34d258dd3db2 */\ +/* sqr(t[0], t[0]); */ /* 125: 1a0111ea397fe69a4b1ba7b64 */\ +/* sqr(t[0], t[0]); */ /* 126: 340223d472ffcd3496374f6c8 */\ +/* sqr(t[0], t[0]); */ /* 127: 680447a8e5ff9a692c6e9ed90 */\ +/* sqr(t[0], t[0]); */ /* 128: d0088f51cbff34d258dd3db20 */\ +/* sqr(t[0], t[0]); */ /* 129: 1a0111ea397fe69a4b1ba7b640 */\ +/* sqr(t[0], t[0]); */ /* 130: 340223d472ffcd3496374f6c80 */\ +/* sqr(t[0], t[0]); */ /* 131: 680447a8e5ff9a692c6e9ed900 */\ +sqr_n_mul(t[0], t[0], 8, t[4]); /* 132: 680447a8e5ff9a692c6e9ed90d */\ +/* sqr(t[0], t[0]); */ /* 133: d0088f51cbff34d258dd3db21a */\ +/* sqr(t[0], t[0]); */ /* 134: 1a0111ea397fe69a4b1ba7b6434 */\ +/* sqr(t[0], t[0]); */ /* 135: 340223d472ffcd3496374f6c868 */\ +/* sqr(t[0], t[0]); */ /* 136: 680447a8e5ff9a692c6e9ed90d0 */\ +/* sqr(t[0], t[0]); */ /* 137: d0088f51cbff34d258dd3db21a0 */\ +/* sqr(t[0], t[0]); */ /* 138: 1a0111ea397fe69a4b1ba7b64340 */\ +/* sqr(t[0], t[0]); */ /* 139: 340223d472ffcd3496374f6c8680 */\ +sqr_n_mul(t[0], t[0], 7, t[12]); /* 140: 340223d472ffcd3496374f6c8697 */\ +/* sqr(t[0], t[0]); */ /* 141: 680447a8e5ff9a692c6e9ed90d2e */\ +/* sqr(t[0], t[0]); */ /* 142: d0088f51cbff34d258dd3db21a5c */\ +/* sqr(t[0], t[0]); */ /* 143: 1a0111ea397fe69a4b1ba7b6434b8 */\ +/* sqr(t[0], t[0]); */ /* 144: 340223d472ffcd3496374f6c86970 */\ +/* sqr(t[0], t[0]); */ /* 145: 680447a8e5ff9a692c6e9ed90d2e0 */\ +sqr_n_mul(t[0], t[0], 5, t[13]); /* 146: 680447a8e5ff9a692c6e9ed90d2eb */\ +/* sqr(t[0], t[0]); */ /* 147: d0088f51cbff34d258dd3db21a5d6 */\ +/* sqr(t[0], t[0]); */ /* 148: 1a0111ea397fe69a4b1ba7b6434bac */\ +/* sqr(t[0], t[0]); */ /* 149: 340223d472ffcd3496374f6c869758 */\ +/* sqr(t[0], t[0]); */ /* 150: 680447a8e5ff9a692c6e9ed90d2eb0 */\ +/* sqr(t[0], t[0]); */ /* 151: d0088f51cbff34d258dd3db21a5d60 */\ +/* sqr(t[0], t[0]); */ /* 152: 1a0111ea397fe69a4b1ba7b6434bac0 */\ +sqr_n_mul(t[0], t[0], 6, t[4]); /* 153: 1a0111ea397fe69a4b1ba7b6434bacd */\ +/* sqr(t[0], t[0]); */ /* 154: 340223d472ffcd3496374f6c869759a */\ +/* sqr(t[0], t[0]); */ /* 155: 680447a8e5ff9a692c6e9ed90d2eb34 */\ +/* sqr(t[0], t[0]); */ /* 156: d0088f51cbff34d258dd3db21a5d668 */\ +/* sqr(t[0], t[0]); */ /* 157: 1a0111ea397fe69a4b1ba7b6434bacd0 */\ +/* sqr(t[0], t[0]); */ /* 158: 340223d472ffcd3496374f6c869759a0 */\ +/* sqr(t[0], t[0]); */ /* 159: 680447a8e5ff9a692c6e9ed90d2eb340 */\ +sqr_n_mul(t[0], t[0], 6, t[6]); /* 160: 680447a8e5ff9a692c6e9ed90d2eb35d */\ +/* sqr(t[0], t[0]); */ /* 161: d0088f51cbff34d258dd3db21a5d66ba */\ +/* sqr(t[0], t[0]); */ /* 162: 1a0111ea397fe69a4b1ba7b6434bacd74 */\ +/* sqr(t[0], t[0]); */ /* 163: 340223d472ffcd3496374f6c869759ae8 */\ +/* sqr(t[0], t[0]); */ /* 164: 680447a8e5ff9a692c6e9ed90d2eb35d0 */\ +sqr_n_mul(t[0], t[0], 4, t[10]); /* 165: 680447a8e5ff9a692c6e9ed90d2eb35d9 */\ +/* sqr(t[0], t[0]); */ /* 166: d0088f51cbff34d258dd3db21a5d66bb2 */\ +/* sqr(t[0], t[0]); */ /* 167: 1a0111ea397fe69a4b1ba7b6434bacd764 */\ +/* sqr(t[0], t[0]); */ /* 168: 340223d472ffcd3496374f6c869759aec8 */\ +/* sqr(t[0], t[0]); */ /* 169: 680447a8e5ff9a692c6e9ed90d2eb35d90 */\ +/* sqr(t[0], t[0]); */ /* 170: d0088f51cbff34d258dd3db21a5d66bb20 */\ +/* sqr(t[0], t[0]); */ /* 171: 1a0111ea397fe69a4b1ba7b6434bacd7640 */\ +/* sqr(t[0], t[0]); */ /* 172: 340223d472ffcd3496374f6c869759aec80 */\ +/* sqr(t[0], t[0]); */ /* 173: 680447a8e5ff9a692c6e9ed90d2eb35d900 */\ +sqr_n_mul(t[0], t[0], 8, t[6]); /* 174: 680447a8e5ff9a692c6e9ed90d2eb35d91d */\ +/* sqr(t[0], t[0]); */ /* 175: d0088f51cbff34d258dd3db21a5d66bb23a */\ +/* sqr(t[0], t[0]); */ /* 176: 1a0111ea397fe69a4b1ba7b6434bacd76474 */\ +/* sqr(t[0], t[0]); */ /* 177: 340223d472ffcd3496374f6c869759aec8e8 */\ +/* sqr(t[0], t[0]); */ /* 178: 680447a8e5ff9a692c6e9ed90d2eb35d91d0 */\ +sqr_n_mul(t[0], t[0], 4, t[4]); /* 179: 680447a8e5ff9a692c6e9ed90d2eb35d91dd */\ +/* sqr(t[0], t[0]); */ /* 180: d0088f51cbff34d258dd3db21a5d66bb23ba */\ +/* sqr(t[0], t[0]); */ /* 181: 1a0111ea397fe69a4b1ba7b6434bacd764774 */\ +/* sqr(t[0], t[0]); */ /* 182: 340223d472ffcd3496374f6c869759aec8ee8 */\ +/* sqr(t[0], t[0]); */ /* 183: 680447a8e5ff9a692c6e9ed90d2eb35d91dd0 */\ +/* sqr(t[0], t[0]); */ /* 184: d0088f51cbff34d258dd3db21a5d66bb23ba0 */\ +/* sqr(t[0], t[0]); */ /* 185: 1a0111ea397fe69a4b1ba7b6434bacd7647740 */\ +/* sqr(t[0], t[0]); */ /* 186: 340223d472ffcd3496374f6c869759aec8ee80 */\ +sqr_n_mul(t[0], t[0], 7, t[12]); /* 187: 340223d472ffcd3496374f6c869759aec8ee97 */\ +/* sqr(t[0], t[0]); */ /* 188: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e */\ +/* sqr(t[0], t[0]); */ /* 189: d0088f51cbff34d258dd3db21a5d66bb23ba5c */\ +/* sqr(t[0], t[0]); */ /* 190: 1a0111ea397fe69a4b1ba7b6434bacd764774b8 */\ +/* sqr(t[0], t[0]); */ /* 191: 340223d472ffcd3496374f6c869759aec8ee970 */\ +/* sqr(t[0], t[0]); */ /* 192: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e0 */\ +/* sqr(t[0], t[0]); */ /* 193: d0088f51cbff34d258dd3db21a5d66bb23ba5c0 */\ +/* sqr(t[0], t[0]); */ /* 194: 1a0111ea397fe69a4b1ba7b6434bacd764774b80 */\ +/* sqr(t[0], t[0]); */ /* 195: 340223d472ffcd3496374f6c869759aec8ee9700 */\ +/* sqr(t[0], t[0]); */ /* 196: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e00 */\ +sqr_n_mul(t[0], t[0], 9, t[11]); /* 197: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13 */\ +/* sqr(t[0], t[0]); */ /* 198: d0088f51cbff34d258dd3db21a5d66bb23ba5c26 */\ +/* sqr(t[0], t[0]); */ /* 199: 1a0111ea397fe69a4b1ba7b6434bacd764774b84c */\ +sqr_n_mul(t[0], t[0], 2, t[9]); /* 200: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f */\ +/* sqr(t[0], t[0]); */ /* 201: 340223d472ffcd3496374f6c869759aec8ee9709e */\ +/* sqr(t[0], t[0]); */ /* 202: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13c */\ +/* sqr(t[0], t[0]); */ /* 203: d0088f51cbff34d258dd3db21a5d66bb23ba5c278 */\ +/* sqr(t[0], t[0]); */ /* 204: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f0 */\ +/* sqr(t[0], t[0]); */ /* 205: 340223d472ffcd3496374f6c869759aec8ee9709e0 */\ +sqr_n_mul(t[0], t[0], 5, t[7]); /* 206: 340223d472ffcd3496374f6c869759aec8ee9709e7 */\ +/* sqr(t[0], t[0]); */ /* 207: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce */\ +/* sqr(t[0], t[0]); */ /* 208: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c */\ +/* sqr(t[0], t[0]); */ /* 209: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38 */\ +/* sqr(t[0], t[0]); */ /* 210: 340223d472ffcd3496374f6c869759aec8ee9709e70 */\ +/* sqr(t[0], t[0]); */ /* 211: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce0 */\ +/* sqr(t[0], t[0]); */ /* 212: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c0 */\ +/* sqr(t[0], t[0]); */ /* 213: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f380 */\ +sqr_n_mul(t[0], t[0], 7, t[2]); /* 214: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f385 */\ +/* sqr(t[0], t[0]); */ /* 215: 340223d472ffcd3496374f6c869759aec8ee9709e70a */\ +/* sqr(t[0], t[0]); */ /* 216: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce14 */\ +/* sqr(t[0], t[0]); */ /* 217: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c28 */\ +/* sqr(t[0], t[0]); */ /* 218: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f3850 */\ +/* sqr(t[0], t[0]); */ /* 219: 340223d472ffcd3496374f6c869759aec8ee9709e70a0 */\ +/* sqr(t[0], t[0]); */ /* 220: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce140 */\ +/* sqr(t[0], t[0]); */ /* 221: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c280 */\ +sqr_n_mul(t[0], t[0], 7, t[10]); /* 222: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c289 */\ +/* sqr(t[0], t[0]); */ /* 223: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512 */\ +/* sqr(t[0], t[0]); */ /* 224: 340223d472ffcd3496374f6c869759aec8ee9709e70a24 */\ +/* sqr(t[0], t[0]); */ /* 225: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce1448 */\ +/* sqr(t[0], t[0]); */ /* 226: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2890 */\ +/* sqr(t[0], t[0]); */ /* 227: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f385120 */\ +/* sqr(t[0], t[0]); */ /* 228: 340223d472ffcd3496374f6c869759aec8ee9709e70a240 */\ +sqr_n_mul(t[0], t[0], 6, t[12]); /* 229: 340223d472ffcd3496374f6c869759aec8ee9709e70a257 */\ +/* sqr(t[0], t[0]); */ /* 230: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144ae */\ +/* sqr(t[0], t[0]); */ /* 231: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895c */\ +/* sqr(t[0], t[0]); */ /* 232: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512b8 */\ +/* sqr(t[0], t[0]); */ /* 233: 340223d472ffcd3496374f6c869759aec8ee9709e70a2570 */\ +/* sqr(t[0], t[0]); */ /* 234: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144ae0 */\ +sqr_n_mul(t[0], t[0], 5, t[6]); /* 235: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd */\ +/* sqr(t[0], t[0]); */ /* 236: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fa */\ +/* sqr(t[0], t[0]); */ /* 237: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf4 */\ +/* sqr(t[0], t[0]); */ /* 238: 340223d472ffcd3496374f6c869759aec8ee9709e70a257e8 */\ +/* sqr(t[0], t[0]); */ /* 239: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd0 */\ +/* sqr(t[0], t[0]); */ /* 240: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fa0 */\ +sqr_n_mul(t[0], t[0], 5, t[11]); /* 241: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3 */\ +/* sqr(t[0], t[0]); */ /* 242: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf66 */\ +/* sqr(t[0], t[0]); */ /* 243: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ecc */\ +/* sqr(t[0], t[0]); */ /* 244: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd98 */\ +/* sqr(t[0], t[0]); */ /* 245: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb30 */\ +/* sqr(t[0], t[0]); */ /* 246: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf660 */\ +sqr_n_mul(t[0], t[0], 5, t[11]); /* 247: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf673 */\ +/* sqr(t[0], t[0]); */ /* 248: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece6 */\ +/* sqr(t[0], t[0]); */ /* 249: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc */\ +/* sqr(t[0], t[0]); */ /* 250: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398 */\ +/* sqr(t[0], t[0]); */ /* 251: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730 */\ +/* sqr(t[0], t[0]); */ /* 252: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece60 */\ +/* sqr(t[0], t[0]); */ /* 253: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc0 */\ +/* sqr(t[0], t[0]); */ /* 254: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3980 */\ +/* sqr(t[0], t[0]); */ /* 255: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf67300 */\ +sqr_n_mul(t[0], t[0], 8, t[4]); /* 256: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d */\ +/* sqr(t[0], t[0]); */ /* 257: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a */\ +/* sqr(t[0], t[0]); */ /* 258: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34 */\ +/* sqr(t[0], t[0]); */ /* 259: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39868 */\ +/* sqr(t[0], t[0]); */ /* 260: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d0 */\ +/* sqr(t[0], t[0]); */ /* 261: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a0 */\ +/* sqr(t[0], t[0]); */ /* 262: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc340 */\ +/* sqr(t[0], t[0]); */ /* 263: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398680 */\ +sqr_n_mul(t[0], t[0], 7, t[3]); /* 264: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398695 */\ +/* sqr(t[0], t[0]); */ /* 265: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a */\ +/* sqr(t[0], t[0]); */ /* 266: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a54 */\ +/* sqr(t[0], t[0]); */ /* 267: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a8 */\ +/* sqr(t[0], t[0]); */ /* 268: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3986950 */\ +/* sqr(t[0], t[0]); */ /* 269: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0 */\ +/* sqr(t[0], t[0]); */ /* 270: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a540 */\ +/* sqr(t[0], t[0]); */ /* 271: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a80 */\ +/* sqr(t[0], t[0]); */ /* 272: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869500 */\ +/* sqr(t[0], t[0]); */ /* 273: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a00 */\ +sqr_n_mul(t[0], t[0], 9, t[8]); /* 274: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f */\ +/* sqr(t[0], t[0]); */ /* 275: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541e */\ +/* sqr(t[0], t[0]); */ /* 276: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83c */\ +/* sqr(t[0], t[0]); */ /* 277: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398695078 */\ +/* sqr(t[0], t[0]); */ /* 278: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f0 */\ +/* sqr(t[0], t[0]); */ /* 279: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541e0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 280: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed */\ +/* sqr(t[0], t[0]); */ /* 281: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83da */\ +/* sqr(t[0], t[0]); */ /* 282: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b4 */\ +/* sqr(t[0], t[0]); */ /* 283: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f68 */\ +sqr_n_mul(t[0], t[0], 3, t[9]); /* 284: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b */\ +/* sqr(t[0], t[0]); */ /* 285: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed6 */\ +/* sqr(t[0], t[0]); */ /* 286: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac */\ +/* sqr(t[0], t[0]); */ /* 287: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b58 */\ +/* sqr(t[0], t[0]); */ /* 288: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0 */\ +/* sqr(t[0], t[0]); */ /* 289: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed60 */\ +/* sqr(t[0], t[0]); */ /* 290: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac0 */\ +/* sqr(t[0], t[0]); */ /* 291: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b580 */\ +/* sqr(t[0], t[0]); */ /* 292: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b00 */\ +sqr_n_mul(t[0], t[0], 8, t[8]); /* 293: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f */\ +/* sqr(t[0], t[0]); */ /* 294: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61e */\ +/* sqr(t[0], t[0]); */ /* 295: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3c */\ +/* sqr(t[0], t[0]); */ /* 296: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b5878 */\ +sqr_n_mul(t[0], t[0], 3, t[9]); /* 297: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b */\ +/* sqr(t[0], t[0]); */ /* 298: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6 */\ +/* sqr(t[0], t[0]); */ /* 299: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec */\ +/* sqr(t[0], t[0]); */ /* 300: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8 */\ +/* sqr(t[0], t[0]); */ /* 301: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b0 */\ +/* sqr(t[0], t[0]); */ /* 302: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f60 */\ +/* sqr(t[0], t[0]); */ /* 303: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec0 */\ +/* sqr(t[0], t[0]); */ /* 304: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d80 */\ +sqr_n_mul(t[0], t[0], 7, t[10]); /* 305: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d89 */\ +/* sqr(t[0], t[0]); */ /* 306: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b12 */\ +/* sqr(t[0], t[0]); */ /* 307: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f624 */\ +/* sqr(t[0], t[0]); */ /* 308: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec48 */\ +/* sqr(t[0], t[0]); */ /* 309: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d890 */\ +/* sqr(t[0], t[0]); */ /* 310: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120 */\ +/* sqr(t[0], t[0]); */ /* 311: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6240 */\ +/* sqr(t[0], t[0]); */ /* 312: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec480 */\ +/* sqr(t[0], t[0]); */ /* 313: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8900 */\ +/* sqr(t[0], t[0]); */ /* 314: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b1200 */\ +sqr_n_mul(t[0], t[0], 9, t[8]); /* 315: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f */\ +/* sqr(t[0], t[0]); */ /* 316: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241e */\ +/* sqr(t[0], t[0]); */ /* 317: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483c */\ +/* sqr(t[0], t[0]); */ /* 318: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d89078 */\ +/* sqr(t[0], t[0]); */ /* 319: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f0 */\ +/* sqr(t[0], t[0]); */ /* 320: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241e0 */\ +/* sqr(t[0], t[0]); */ /* 321: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483c0 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 322: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d5 */\ +/* sqr(t[0], t[0]); */ /* 323: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aa */\ +/* sqr(t[0], t[0]); */ /* 324: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f54 */\ +/* sqr(t[0], t[0]); */ /* 325: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241ea8 */\ +/* sqr(t[0], t[0]); */ /* 326: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d50 */\ +/* sqr(t[0], t[0]); */ /* 327: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aa0 */\ +/* sqr(t[0], t[0]); */ /* 328: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f540 */\ +sqr_n_mul(t[0], t[0], 6, t[5]); /* 329: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55f */\ +/* sqr(t[0], t[0]); */ /* 330: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabe */\ +/* sqr(t[0], t[0]); */ /* 331: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57c */\ +/* sqr(t[0], t[0]); */ /* 332: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaf8 */\ +/* sqr(t[0], t[0]); */ /* 333: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55f0 */\ +/* sqr(t[0], t[0]); */ /* 334: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 335: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabff */\ +/* sqr(t[0], t[0]); */ /* 336: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fe */\ +/* sqr(t[0], t[0]); */ /* 337: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffc */\ +/* sqr(t[0], t[0]); */ /* 338: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ff8 */\ +/* sqr(t[0], t[0]); */ /* 339: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabff0 */\ +/* sqr(t[0], t[0]); */ /* 340: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 341: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fff */\ +/* sqr(t[0], t[0]); */ /* 342: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aafffe */\ +/* sqr(t[0], t[0]); */ /* 343: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55fffc */\ +/* sqr(t[0], t[0]); */ /* 344: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfff8 */\ +/* sqr(t[0], t[0]); */ /* 345: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fff0 */\ +sqr_n_mul(t[0], t[0], 4, t[4]); /* 346: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd */\ +/* sqr(t[0], t[0]); */ /* 347: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffa */\ +/* sqr(t[0], t[0]); */ /* 348: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff4 */\ +/* sqr(t[0], t[0]); */ /* 349: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffe8 */\ +sqr_n_mul(t[0], t[0], 3, t[9]); /* 350: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb */\ +/* sqr(t[0], t[0]); */ /* 351: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd6 */\ +/* sqr(t[0], t[0]); */ /* 352: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac */\ +/* sqr(t[0], t[0]); */ /* 353: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58 */\ +/* sqr(t[0], t[0]); */ /* 354: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb0 */\ +/* sqr(t[0], t[0]); */ /* 355: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd60 */\ +/* sqr(t[0], t[0]); */ /* 356: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac0 */\ +/* sqr(t[0], t[0]); */ /* 357: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff580 */\ +/* sqr(t[0], t[0]); */ /* 358: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb00 */\ +sqr_n_mul(t[0], t[0], 8, t[3]); /* 359: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb15 */\ +/* sqr(t[0], t[0]); */ /* 360: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a */\ +/* sqr(t[0], t[0]); */ /* 361: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54 */\ +/* sqr(t[0], t[0]); */ /* 362: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a8 */\ +/* sqr(t[0], t[0]); */ /* 363: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb150 */\ +/* sqr(t[0], t[0]); */ /* 364: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a0 */\ +/* sqr(t[0], t[0]); */ /* 365: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac540 */\ +/* sqr(t[0], t[0]); */ /* 366: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a80 */\ +sqr_n_mul(t[0], t[0], 7, t[5]); /* 367: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9f */\ +/* sqr(t[0], t[0]); */ /* 368: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153e */\ +/* sqr(t[0], t[0]); */ /* 369: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7c */\ +/* sqr(t[0], t[0]); */ /* 370: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54f8 */\ +/* sqr(t[0], t[0]); */ /* 371: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9f0 */\ +/* sqr(t[0], t[0]); */ /* 372: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153e0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 373: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ff */\ +/* sqr(t[0], t[0]); */ /* 374: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fe */\ +/* sqr(t[0], t[0]); */ /* 375: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffc */\ +/* sqr(t[0], t[0]); */ /* 376: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ff8 */\ +/* sqr(t[0], t[0]); */ /* 377: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ff0 */\ +/* sqr(t[0], t[0]); */ /* 378: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 379: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fff */\ +/* sqr(t[0], t[0]); */ /* 380: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54fffe */\ +/* sqr(t[0], t[0]); */ /* 381: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9fffc */\ +/* sqr(t[0], t[0]); */ /* 382: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153fff8 */\ +/* sqr(t[0], t[0]); */ /* 383: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fff0 */\ +sqr_n_mul(t[0], t[0], 4, t[8]); /* 384: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff */\ +/* sqr(t[0], t[0]); */ /* 385: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffe */\ +/* sqr(t[0], t[0]); */ /* 386: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffc */\ +/* sqr(t[0], t[0]); */ /* 387: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffff8 */\ +/* sqr(t[0], t[0]); */ /* 388: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff0 */\ +sqr_n_mul(t[0], t[0], 4, t[7]); /* 389: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff7 */\ +/* sqr(t[0], t[0]); */ /* 390: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee */\ +/* sqr(t[0], t[0]); */ /* 391: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdc */\ +/* sqr(t[0], t[0]); */ /* 392: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb8 */\ +/* sqr(t[0], t[0]); */ /* 393: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff70 */\ +/* sqr(t[0], t[0]); */ /* 394: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee0 */\ +/* sqr(t[0], t[0]); */ /* 395: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdc0 */\ +/* sqr(t[0], t[0]); */ /* 396: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb80 */\ +sqr_n_mul(t[0], t[0], 7, t[5]); /* 397: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9f */\ +/* sqr(t[0], t[0]); */ /* 398: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73e */\ +/* sqr(t[0], t[0]); */ /* 399: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7c */\ +/* sqr(t[0], t[0]); */ /* 400: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcf8 */\ +/* sqr(t[0], t[0]); */ /* 401: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9f0 */\ +/* sqr(t[0], t[0]); */ /* 402: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73e0 */\ +sqr_n_mul(t[0], t[0], 5, t[6]); /* 403: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fd */\ +/* sqr(t[0], t[0]); */ /* 404: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fa */\ +/* sqr(t[0], t[0]); */ /* 405: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff4 */\ +/* sqr(t[0], t[0]); */ /* 406: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fe8 */\ +/* sqr(t[0], t[0]); */ /* 407: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fd0 */\ +/* sqr(t[0], t[0]); */ /* 408: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fa0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 409: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbf */\ +/* sqr(t[0], t[0]); */ /* 410: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7e */\ +/* sqr(t[0], t[0]); */ /* 411: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefc */\ +/* sqr(t[0], t[0]); */ /* 412: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdf8 */\ +/* sqr(t[0], t[0]); */ /* 413: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbf0 */\ +/* sqr(t[0], t[0]); */ /* 414: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7e0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 415: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ff */\ +/* sqr(t[0], t[0]); */ /* 416: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffe */\ +/* sqr(t[0], t[0]); */ /* 417: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffc */\ +/* sqr(t[0], t[0]); */ /* 418: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbff8 */\ +/* sqr(t[0], t[0]); */ /* 419: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ff0 */\ +/* sqr(t[0], t[0]); */ /* 420: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 421: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffff */\ +/* sqr(t[0], t[0]); */ /* 422: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffe */\ +/* sqr(t[0], t[0]); */ /* 423: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffc */\ +/* sqr(t[0], t[0]); */ /* 424: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fff8 */\ +/* sqr(t[0], t[0]); */ /* 425: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffff0 */\ +/* sqr(t[0], t[0]); */ /* 426: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 427: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffff */\ +/* sqr(t[0], t[0]); */ /* 428: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffe */\ +/* sqr(t[0], t[0]); */ /* 429: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ffffc */\ +/* sqr(t[0], t[0]); */ /* 430: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefffff8 */\ +/* sqr(t[0], t[0]); */ /* 431: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffff0 */\ +/* sqr(t[0], t[0]); */ /* 432: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 433: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffff */\ +/* sqr(t[0], t[0]); */ /* 434: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffe */\ +/* sqr(t[0], t[0]); */ /* 435: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffc */\ +/* sqr(t[0], t[0]); */ /* 436: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffff8 */\ +/* sqr(t[0], t[0]); */ /* 437: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffff0 */\ +/* sqr(t[0], t[0]); */ /* 438: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 439: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffff */\ +/* sqr(t[0], t[0]); */ /* 440: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefffffffe */\ +/* sqr(t[0], t[0]); */ /* 441: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffffffc */\ +/* sqr(t[0], t[0]); */ /* 442: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffff8 */\ +/* sqr(t[0], t[0]); */ /* 443: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffff0 */\ +sqr_n_mul(t[0], t[0], 4, t[4]); /* 444: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd */\ +/* sqr(t[0], t[0]); */ /* 445: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffa */\ +/* sqr(t[0], t[0]); */ /* 446: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff4 */\ +/* sqr(t[0], t[0]); */ /* 447: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffe8 */\ +/* sqr(t[0], t[0]); */ /* 448: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd0 */\ +/* sqr(t[0], t[0]); */ /* 449: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffa0 */\ +/* sqr(t[0], t[0]); */ /* 450: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff40 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 451: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff55 */\ +/* sqr(t[0], t[0]); */ /* 452: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaa */\ +/* sqr(t[0], t[0]); */ /* 453: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd54 */\ +/* sqr(t[0], t[0]); */ /* 454: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaa8 */\ +/* sqr(t[0], t[0]); */ /* 455: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff550 */\ +sqr_n_mul(t[0], t[0], 4, t[2]); /* 456: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff555 */\ +/* sqr(t[0], t[0]); */ /* 457: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaaa */\ +/* sqr(t[0], t[0]); */ /* 458: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd554 */\ +/* sqr(t[0], t[0]); */ /* 459: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaa8 */\ +sqr_n_mul(out, t[0], 3, t[1]); /* 460: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaa9 */\ +} while(0) diff --git a/crypto/blst_src/recip.c b/crypto/blst_src/recip.c new file mode 100644 index 00000000000..e0c700635ed --- /dev/null +++ b/crypto/blst_src/recip.c @@ -0,0 +1,139 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" + +#ifdef __OPTIMIZE_SIZE__ +/* + * 608 multiplications for scalar inversion modulo BLS12-381 prime, 32% + * more than corresponding optimal addition-chain, plus mispredicted + * branch penalties on top of that... The addition chain below was + * measured to be >50% faster. + */ +static void flt_reciprocal_fp(vec384 out, const vec384 inp) +{ + static const byte BLS12_381_P_minus_2[] = { + TO_BYTES(0xb9feffffffffaaa9), TO_BYTES(0x1eabfffeb153ffff), + TO_BYTES(0x6730d2a0f6b0f624), TO_BYTES(0x64774b84f38512bf), + TO_BYTES(0x4b1ba7b6434bacd7), TO_BYTES(0x1a0111ea397fe69a) + }; + + exp_mont_384(out, inp, BLS12_381_P_minus_2, 381, BLS12_381_P, p0); +} +#else +# define sqr(ret,a) sqr_fp(ret,a) +# define mul(ret,a,b) mul_fp(ret,a,b) +# define sqr_n_mul(ret,a,n,b) sqr_n_mul_fp(ret,a,n,b) + +# include "recip-addchain.h" +static void flt_reciprocal_fp(vec384 out, const vec384 inp) +{ + RECIPROCAL_MOD_BLS12_381_P(out, inp, vec384); +} +# undef RECIPROCAL_MOD_BLS12_381_P +# undef sqr_n_mul +# undef mul +# undef sqr +#endif + +static void flt_reciprocal_fp2(vec384x out, const vec384x inp) +{ + vec384 t0, t1; + + /* + * |out| = 1/(a + b*i) = a/(a^2+b^2) - b/(a^2+b^2)*i + */ + sqr_fp(t0, inp[0]); + sqr_fp(t1, inp[1]); + add_fp(t0, t0, t1); + flt_reciprocal_fp(t1, t0); + mul_fp(out[0], inp[0], t1); + mul_fp(out[1], inp[1], t1); + neg_fp(out[1], out[1]); +} + +static void reciprocal_fp(vec384 out, const vec384 inp) +{ + static const vec384 Px8 = { /* left-aligned value of the modulus */ + TO_LIMB_T(0xcff7fffffffd5558), TO_LIMB_T(0xf55ffff58a9ffffd), + TO_LIMB_T(0x39869507b587b120), TO_LIMB_T(0x23ba5c279c2895fb), + TO_LIMB_T(0x58dd3db21a5d66bb), TO_LIMB_T(0xd0088f51cbff34d2) + }; +#ifdef __BLST_NO_ASM__ +# define RRx4 BLS12_381_RR +#else + static const vec384 RRx4 = { /* (4<<768)%P */ + TO_LIMB_T(0x5f7e7cd070d107c2), TO_LIMB_T(0xec839a9ac49c13c8), + TO_LIMB_T(0x6933786f44f4ef0b), TO_LIMB_T(0xd6bf8b9c676be983), + TO_LIMB_T(0xd3adaaaa4dcefb06), TO_LIMB_T(0x12601bc1d82bc175) + }; +#endif + union { vec768 x; vec384 r[2]; } temp; + + ct_inverse_mod_383(temp.x, inp, BLS12_381_P, Px8); + redc_mont_384(temp.r[0], temp.x, BLS12_381_P, p0); + mul_mont_384(temp.r[0], temp.r[0], RRx4, BLS12_381_P, p0); + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* sign goes straight to flt_reciprocal */ + mul_mont_384(temp.r[1], temp.r[0], inp, BLS12_381_P, p0); + if (vec_is_equal(temp.r[1], BLS12_381_Rx.p, sizeof(vec384)) | + vec_is_zero(temp.r[1], sizeof(vec384))) + vec_copy(out, temp.r[0], sizeof(vec384)); + else + flt_reciprocal_fp(out, inp); +#else + vec_copy(out, temp.r[0], sizeof(vec384)); +#endif +#undef RRx4 +} + +void blst_fp_inverse(vec384 out, const vec384 inp) +{ reciprocal_fp(out, inp); } + +void blst_fp_eucl_inverse(vec384 ret, const vec384 a) +{ reciprocal_fp(ret, a); } + +static void reciprocal_fp2(vec384x out, const vec384x inp) +{ + vec384 t0, t1; + + /* + * |out| = 1/(a + b*i) = a/(a^2+b^2) - b/(a^2+b^2)*i + */ + sqr_fp(t0, inp[0]); + sqr_fp(t1, inp[1]); + add_fp(t0, t0, t1); + reciprocal_fp(t1, t0); + mul_fp(out[0], inp[0], t1); + mul_fp(out[1], inp[1], t1); + neg_fp(out[1], out[1]); +} + +void blst_fp2_inverse(vec384x out, const vec384x inp) +{ reciprocal_fp2(out, inp); } + +void blst_fp2_eucl_inverse(vec384x out, const vec384x inp) +{ reciprocal_fp2(out, inp); } + +static void reciprocal_fr(vec256 out, const vec256 inp) +{ + static const vec256 rx2 = { /* left-aligned value of the modulus */ + TO_LIMB_T(0xfffffffe00000002), TO_LIMB_T(0xa77b4805fffcb7fd), + TO_LIMB_T(0x6673b0101343b00a), TO_LIMB_T(0xe7db4ea6533afa90), + }; + vec512 temp; + + ct_inverse_mod_256(temp, inp, BLS12_381_r, rx2); + redc_mont_256(out, temp, BLS12_381_r, r0); + mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); +} + +void blst_fr_inverse(vec256 out, const vec256 inp) +{ reciprocal_fr(out, inp); } + +void blst_fr_eucl_inverse(vec256 out, const vec256 inp) +{ reciprocal_fr(out, inp); } diff --git a/crypto/blst_src/sha256.h b/crypto/blst_src/sha256.h new file mode 100644 index 00000000000..77ddb6dc848 --- /dev/null +++ b/crypto/blst_src/sha256.h @@ -0,0 +1,140 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_SHA256_H__ +#define __BLS12_381_ASM_SHA256_H__ + +#include "vect.h" + +#if (defined(__x86_64__) || defined(__x86_64) || defined(_M_X64)) && \ + defined(__SHA__) /* -msha */ && !defined(__BLST_PORTABLE__) +# define sha256_block_data_order blst_sha256_block_data_order_shaext +#elif defined(__aarch64__) && \ + defined(__ARM_FEATURE_CRYPTO) && !defined(__BLST_PORTABLE__) +# define sha256_block_data_order blst_sha256_block_armv8 +#else +# define sha256_block_data_order blst_sha256_block_data_order +#endif +#define sha256_hcopy blst_sha256_hcopy +#define sha256_bcopy blst_sha256_bcopy +#define sha256_emit blst_sha256_emit + +void sha256_block_data_order(unsigned int *h, const void *inp, size_t blocks); +void sha256_hcopy(unsigned int dst[8], const unsigned int src[8]); +void sha256_bcopy(void *dst, const void *src, size_t len); + +/* + * If SHA256_CTX conflicts with something, just redefine it to alternative + * custom name prior including this header. + */ +typedef struct { + unsigned int h[8]; + unsigned long long N; + unsigned char buf[64]; + size_t off; +} SHA256_CTX; + + +static void sha256_init_h(unsigned int h[8]) +{ + h[0] = 0x6a09e667U; + h[1] = 0xbb67ae85U; + h[2] = 0x3c6ef372U; + h[3] = 0xa54ff53aU; + h[4] = 0x510e527fU; + h[5] = 0x9b05688cU; + h[6] = 0x1f83d9abU; + h[7] = 0x5be0cd19U; +} + +static void sha256_init(SHA256_CTX *ctx) +{ + sha256_init_h(ctx->h); + ctx->N = 0; + vec_zero(ctx->buf, sizeof(ctx->buf)); + ctx->off = 0; +} + +static void sha256_update(SHA256_CTX *ctx, const void *_inp, size_t len) +{ + size_t n; + const unsigned char *inp = _inp; + + ctx->N += len; + + if ((len != 0) & ((n = ctx->off) != 0)) { + size_t rem = sizeof(ctx->buf) - n; + + if (rem > len) { + sha256_bcopy(ctx->buf + n, inp, len); + ctx->off += len; + return; + } else { + sha256_bcopy(ctx->buf + n, inp, rem); + inp += rem; + len -= rem; + sha256_block_data_order(ctx->h, ctx->buf, 1); + vec_zero(ctx->buf, sizeof(ctx->buf)); + ctx->off = 0; + } + } + + n = len / sizeof(ctx->buf); + if (n > 0) { + sha256_block_data_order(ctx->h, inp, n); + n *= sizeof(ctx->buf); + inp += n; + len -= n; + } + + if (len) + sha256_bcopy(ctx->buf, inp, ctx->off = len); +} + +#define __TOBE32(ptr, val) ((ptr)[0] = (unsigned char)((val)>>24), \ + (ptr)[1] = (unsigned char)((val)>>16), \ + (ptr)[2] = (unsigned char)((val)>>8), \ + (ptr)[3] = (unsigned char)(val)) + +#if 1 +void sha256_emit(unsigned char md[32], const unsigned int h[8]); +#else +static void sha256_emit(unsigned char md[32], const unsigned int h[8]) +{ + unsigned int h_i; + + h_i = h[0]; __TOBE32(md + 0, h_i); + h_i = h[1]; __TOBE32(md + 4, h_i); + h_i = h[2]; __TOBE32(md + 8, h_i); + h_i = h[3]; __TOBE32(md + 12, h_i); + h_i = h[4]; __TOBE32(md + 16, h_i); + h_i = h[5]; __TOBE32(md + 20, h_i); + h_i = h[6]; __TOBE32(md + 24, h_i); + h_i = h[7]; __TOBE32(md + 28, h_i); +} +#endif + +static void sha256_final(unsigned char md[32], SHA256_CTX *ctx) +{ + unsigned long long bits = ctx->N * 8; + size_t n = ctx->off; + unsigned char *tail; + + ctx->buf[n++] = 0x80; + + if (n > (sizeof(ctx->buf) - 8)) { + sha256_block_data_order(ctx->h, ctx->buf, 1); + vec_zero(ctx->buf, sizeof(ctx->buf)); + } + + tail = ctx->buf + sizeof(ctx->buf) - 8; + __TOBE32(tail, (unsigned int)(bits >> 32)); + __TOBE32(tail + 4, (unsigned int)bits); + sha256_block_data_order(ctx->h, ctx->buf, 1); + sha256_emit(md, ctx->h); +} + +#undef __TOBE32 +#endif diff --git a/crypto/blst_src/sqrt-addchain.h b/crypto/blst_src/sqrt-addchain.h new file mode 100644 index 00000000000..4e7f0beb6b1 --- /dev/null +++ b/crypto/blst_src/sqrt-addchain.h @@ -0,0 +1,489 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * The "magic" number is (BLS12_381_P-3)/4. Exponentiation to which + * yields reciprocal of sqrt(x), which is used in simplified Shallue- + * van de Woestijne-Ulas map-to-curve method, but it's trivial to adapt + * it for more "traditional" sqrt(x) as 'x*ret' (or for is_square(x) + * as 'x*ret^2==1'). + * + * Generated with 'addchain 1000602388805416848354447456433976039139220704984751971333014534031007912622709466110671907282253916009473568139946' + * https://github.com/kwantam/addchain + * + * # Bos-Coster (win=4) : 458 (16) <<< + * # Bos-Coster (win=5) : 460 (28) + * # Bos-Coster (win=6) : 461 (33) + * # Bos-Coster (win=7) : 460 (28) + * # Bos-Coster (win=3) : 462 ( 9) + * # Bos-Coster (win=8) : 466 (34) + * # Bos-Coster (win=9) : 464 (31) + * # Yacobi : 478 (31) + * # Bos-Coster (win=10) : 473 (30) + * # Bos-Coster (win=2) : 486 ( 5) + * # Bergeron-Berstel-Brlek-Duboc : 489 ( 5) + */ + +#define RECIP_SQRT_MOD_BLS12_381_P(out, inp, ptype) do { \ +ptype t[16]; \ +vec_copy(t[13], inp, sizeof(ptype));/* 0: 1 */\ +sqr(t[0], t[13]); /* 1: 2 */\ +mul(t[8], t[0], t[13]); /* 2: 3 */\ +sqr(t[4], t[0]); /* 3: 4 */\ +mul(t[1], t[8], t[0]); /* 4: 5 */\ +mul(t[6], t[4], t[8]); /* 5: 7 */\ +mul(t[9], t[1], t[4]); /* 6: 9 */\ +mul(t[12], t[6], t[4]); /* 7: b */\ +mul(t[3], t[9], t[4]); /* 8: d */\ +mul(t[7], t[12], t[4]); /* 9: f */\ +mul(t[15], t[3], t[4]); /* 10: 11 */\ +mul(t[10], t[7], t[4]); /* 11: 13 */\ +mul(t[2], t[15], t[4]); /* 12: 15 */\ +mul(t[11], t[10], t[4]); /* 13: 17 */\ +sqr(t[0], t[3]); /* 14: 1a */\ +mul(t[14], t[11], t[4]); /* 15: 1b */\ +mul(t[5], t[0], t[8]); /* 16: 1d */\ +mul(t[4], t[0], t[1]); /* 17: 1f */\ +/* sqr(t[0], t[0]); */ /* 18: 34 */\ +/* sqr(t[0], t[0]); */ /* 19: 68 */\ +/* sqr(t[0], t[0]); */ /* 20: d0 */\ +/* sqr(t[0], t[0]); */ /* 21: 1a0 */\ +/* sqr(t[0], t[0]); */ /* 22: 340 */\ +/* sqr(t[0], t[0]); */ /* 23: 680 */\ +/* sqr(t[0], t[0]); */ /* 24: d00 */\ +/* sqr(t[0], t[0]); */ /* 25: 1a00 */\ +/* sqr(t[0], t[0]); */ /* 26: 3400 */\ +/* sqr(t[0], t[0]); */ /* 27: 6800 */\ +/* sqr(t[0], t[0]); */ /* 28: d000 */\ +/* sqr(t[0], t[0]); */ /* 29: 1a000 */\ +sqr_n_mul(t[0], t[0], 12, t[15]); /* 30: 1a011 */\ +/* sqr(t[0], t[0]); */ /* 31: 34022 */\ +/* sqr(t[0], t[0]); */ /* 32: 68044 */\ +/* sqr(t[0], t[0]); */ /* 33: d0088 */\ +/* sqr(t[0], t[0]); */ /* 34: 1a0110 */\ +/* sqr(t[0], t[0]); */ /* 35: 340220 */\ +/* sqr(t[0], t[0]); */ /* 36: 680440 */\ +/* sqr(t[0], t[0]); */ /* 37: d00880 */\ +sqr_n_mul(t[0], t[0], 7, t[7]); /* 38: d0088f */\ +/* sqr(t[0], t[0]); */ /* 39: 1a0111e */\ +/* sqr(t[0], t[0]); */ /* 40: 340223c */\ +/* sqr(t[0], t[0]); */ /* 41: 6804478 */\ +/* sqr(t[0], t[0]); */ /* 42: d0088f0 */\ +sqr_n_mul(t[0], t[0], 4, t[1]); /* 43: d0088f5 */\ +/* sqr(t[0], t[0]); */ /* 44: 1a0111ea */\ +/* sqr(t[0], t[0]); */ /* 45: 340223d4 */\ +/* sqr(t[0], t[0]); */ /* 46: 680447a8 */\ +/* sqr(t[0], t[0]); */ /* 47: d0088f50 */\ +/* sqr(t[0], t[0]); */ /* 48: 1a0111ea0 */\ +/* sqr(t[0], t[0]); */ /* 49: 340223d40 */\ +sqr_n_mul(t[0], t[0], 6, t[6]); /* 50: 340223d47 */\ +/* sqr(t[0], t[0]); */ /* 51: 680447a8e */\ +/* sqr(t[0], t[0]); */ /* 52: d0088f51c */\ +/* sqr(t[0], t[0]); */ /* 53: 1a0111ea38 */\ +/* sqr(t[0], t[0]); */ /* 54: 340223d470 */\ +/* sqr(t[0], t[0]); */ /* 55: 680447a8e0 */\ +/* sqr(t[0], t[0]); */ /* 56: d0088f51c0 */\ +/* sqr(t[0], t[0]); */ /* 57: 1a0111ea380 */\ +sqr_n_mul(t[0], t[0], 7, t[11]); /* 58: 1a0111ea397 */\ +/* sqr(t[0], t[0]); */ /* 59: 340223d472e */\ +/* sqr(t[0], t[0]); */ /* 60: 680447a8e5c */\ +/* sqr(t[0], t[0]); */ /* 61: d0088f51cb8 */\ +/* sqr(t[0], t[0]); */ /* 62: 1a0111ea3970 */\ +/* sqr(t[0], t[0]); */ /* 63: 340223d472e0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 64: 340223d472ff */\ +/* sqr(t[0], t[0]); */ /* 65: 680447a8e5fe */\ +/* sqr(t[0], t[0]); */ /* 66: d0088f51cbfc */\ +sqr_n_mul(t[0], t[0], 2, t[8]); /* 67: d0088f51cbff */\ +/* sqr(t[0], t[0]); */ /* 68: 1a0111ea397fe */\ +/* sqr(t[0], t[0]); */ /* 69: 340223d472ffc */\ +/* sqr(t[0], t[0]); */ /* 70: 680447a8e5ff8 */\ +/* sqr(t[0], t[0]); */ /* 71: d0088f51cbff0 */\ +/* sqr(t[0], t[0]); */ /* 72: 1a0111ea397fe0 */\ +/* sqr(t[0], t[0]); */ /* 73: 340223d472ffc0 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 74: 340223d472ffcd */\ +/* sqr(t[0], t[0]); */ /* 75: 680447a8e5ff9a */\ +/* sqr(t[0], t[0]); */ /* 76: d0088f51cbff34 */\ +/* sqr(t[0], t[0]); */ /* 77: 1a0111ea397fe68 */\ +/* sqr(t[0], t[0]); */ /* 78: 340223d472ffcd0 */\ +/* sqr(t[0], t[0]); */ /* 79: 680447a8e5ff9a0 */\ +/* sqr(t[0], t[0]); */ /* 80: d0088f51cbff340 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 81: d0088f51cbff34d */\ +/* sqr(t[0], t[0]); */ /* 82: 1a0111ea397fe69a */\ +/* sqr(t[0], t[0]); */ /* 83: 340223d472ffcd34 */\ +/* sqr(t[0], t[0]); */ /* 84: 680447a8e5ff9a68 */\ +/* sqr(t[0], t[0]); */ /* 85: d0088f51cbff34d0 */\ +/* sqr(t[0], t[0]); */ /* 86: 1a0111ea397fe69a0 */\ +/* sqr(t[0], t[0]); */ /* 87: 340223d472ffcd340 */\ +sqr_n_mul(t[0], t[0], 6, t[9]); /* 88: 340223d472ffcd349 */\ +/* sqr(t[0], t[0]); */ /* 89: 680447a8e5ff9a692 */\ +/* sqr(t[0], t[0]); */ /* 90: d0088f51cbff34d24 */\ +/* sqr(t[0], t[0]); */ /* 91: 1a0111ea397fe69a48 */\ +sqr_n_mul(t[0], t[0], 3, t[8]); /* 92: 1a0111ea397fe69a4b */\ +/* sqr(t[0], t[0]); */ /* 93: 340223d472ffcd3496 */\ +/* sqr(t[0], t[0]); */ /* 94: 680447a8e5ff9a692c */\ +/* sqr(t[0], t[0]); */ /* 95: d0088f51cbff34d258 */\ +/* sqr(t[0], t[0]); */ /* 96: 1a0111ea397fe69a4b0 */\ +/* sqr(t[0], t[0]); */ /* 97: 340223d472ffcd34960 */\ +/* sqr(t[0], t[0]); */ /* 98: 680447a8e5ff9a692c0 */\ +/* sqr(t[0], t[0]); */ /* 99: d0088f51cbff34d2580 */\ +sqr_n_mul(t[0], t[0], 7, t[3]); /* 100: d0088f51cbff34d258d */\ +/* sqr(t[0], t[0]); */ /* 101: 1a0111ea397fe69a4b1a */\ +/* sqr(t[0], t[0]); */ /* 102: 340223d472ffcd349634 */\ +/* sqr(t[0], t[0]); */ /* 103: 680447a8e5ff9a692c68 */\ +/* sqr(t[0], t[0]); */ /* 104: d0088f51cbff34d258d0 */\ +sqr_n_mul(t[0], t[0], 4, t[3]); /* 105: d0088f51cbff34d258dd */\ +/* sqr(t[0], t[0]); */ /* 106: 1a0111ea397fe69a4b1ba */\ +/* sqr(t[0], t[0]); */ /* 107: 340223d472ffcd3496374 */\ +/* sqr(t[0], t[0]); */ /* 108: 680447a8e5ff9a692c6e8 */\ +/* sqr(t[0], t[0]); */ /* 109: d0088f51cbff34d258dd0 */\ +/* sqr(t[0], t[0]); */ /* 110: 1a0111ea397fe69a4b1ba0 */\ +/* sqr(t[0], t[0]); */ /* 111: 340223d472ffcd34963740 */\ +sqr_n_mul(t[0], t[0], 6, t[7]); /* 112: 340223d472ffcd3496374f */\ +/* sqr(t[0], t[0]); */ /* 113: 680447a8e5ff9a692c6e9e */\ +/* sqr(t[0], t[0]); */ /* 114: d0088f51cbff34d258dd3c */\ +/* sqr(t[0], t[0]); */ /* 115: 1a0111ea397fe69a4b1ba78 */\ +/* sqr(t[0], t[0]); */ /* 116: 340223d472ffcd3496374f0 */\ +/* sqr(t[0], t[0]); */ /* 117: 680447a8e5ff9a692c6e9e0 */\ +/* sqr(t[0], t[0]); */ /* 118: d0088f51cbff34d258dd3c0 */\ +sqr_n_mul(t[0], t[0], 6, t[14]); /* 119: d0088f51cbff34d258dd3db */\ +/* sqr(t[0], t[0]); */ /* 120: 1a0111ea397fe69a4b1ba7b6 */\ +/* sqr(t[0], t[0]); */ /* 121: 340223d472ffcd3496374f6c */\ +/* sqr(t[0], t[0]); */ /* 122: 680447a8e5ff9a692c6e9ed8 */\ +sqr_n_mul(t[0], t[0], 3, t[13]); /* 123: 680447a8e5ff9a692c6e9ed9 */\ +/* sqr(t[0], t[0]); */ /* 124: d0088f51cbff34d258dd3db2 */\ +/* sqr(t[0], t[0]); */ /* 125: 1a0111ea397fe69a4b1ba7b64 */\ +/* sqr(t[0], t[0]); */ /* 126: 340223d472ffcd3496374f6c8 */\ +/* sqr(t[0], t[0]); */ /* 127: 680447a8e5ff9a692c6e9ed90 */\ +/* sqr(t[0], t[0]); */ /* 128: d0088f51cbff34d258dd3db20 */\ +/* sqr(t[0], t[0]); */ /* 129: 1a0111ea397fe69a4b1ba7b640 */\ +/* sqr(t[0], t[0]); */ /* 130: 340223d472ffcd3496374f6c80 */\ +/* sqr(t[0], t[0]); */ /* 131: 680447a8e5ff9a692c6e9ed900 */\ +sqr_n_mul(t[0], t[0], 8, t[3]); /* 132: 680447a8e5ff9a692c6e9ed90d */\ +/* sqr(t[0], t[0]); */ /* 133: d0088f51cbff34d258dd3db21a */\ +/* sqr(t[0], t[0]); */ /* 134: 1a0111ea397fe69a4b1ba7b6434 */\ +/* sqr(t[0], t[0]); */ /* 135: 340223d472ffcd3496374f6c868 */\ +/* sqr(t[0], t[0]); */ /* 136: 680447a8e5ff9a692c6e9ed90d0 */\ +/* sqr(t[0], t[0]); */ /* 137: d0088f51cbff34d258dd3db21a0 */\ +/* sqr(t[0], t[0]); */ /* 138: 1a0111ea397fe69a4b1ba7b64340 */\ +/* sqr(t[0], t[0]); */ /* 139: 340223d472ffcd3496374f6c8680 */\ +sqr_n_mul(t[0], t[0], 7, t[11]); /* 140: 340223d472ffcd3496374f6c8697 */\ +/* sqr(t[0], t[0]); */ /* 141: 680447a8e5ff9a692c6e9ed90d2e */\ +/* sqr(t[0], t[0]); */ /* 142: d0088f51cbff34d258dd3db21a5c */\ +/* sqr(t[0], t[0]); */ /* 143: 1a0111ea397fe69a4b1ba7b6434b8 */\ +/* sqr(t[0], t[0]); */ /* 144: 340223d472ffcd3496374f6c86970 */\ +/* sqr(t[0], t[0]); */ /* 145: 680447a8e5ff9a692c6e9ed90d2e0 */\ +sqr_n_mul(t[0], t[0], 5, t[12]); /* 146: 680447a8e5ff9a692c6e9ed90d2eb */\ +/* sqr(t[0], t[0]); */ /* 147: d0088f51cbff34d258dd3db21a5d6 */\ +/* sqr(t[0], t[0]); */ /* 148: 1a0111ea397fe69a4b1ba7b6434bac */\ +/* sqr(t[0], t[0]); */ /* 149: 340223d472ffcd3496374f6c869758 */\ +/* sqr(t[0], t[0]); */ /* 150: 680447a8e5ff9a692c6e9ed90d2eb0 */\ +/* sqr(t[0], t[0]); */ /* 151: d0088f51cbff34d258dd3db21a5d60 */\ +/* sqr(t[0], t[0]); */ /* 152: 1a0111ea397fe69a4b1ba7b6434bac0 */\ +sqr_n_mul(t[0], t[0], 6, t[3]); /* 153: 1a0111ea397fe69a4b1ba7b6434bacd */\ +/* sqr(t[0], t[0]); */ /* 154: 340223d472ffcd3496374f6c869759a */\ +/* sqr(t[0], t[0]); */ /* 155: 680447a8e5ff9a692c6e9ed90d2eb34 */\ +/* sqr(t[0], t[0]); */ /* 156: d0088f51cbff34d258dd3db21a5d668 */\ +/* sqr(t[0], t[0]); */ /* 157: 1a0111ea397fe69a4b1ba7b6434bacd0 */\ +/* sqr(t[0], t[0]); */ /* 158: 340223d472ffcd3496374f6c869759a0 */\ +/* sqr(t[0], t[0]); */ /* 159: 680447a8e5ff9a692c6e9ed90d2eb340 */\ +sqr_n_mul(t[0], t[0], 6, t[5]); /* 160: 680447a8e5ff9a692c6e9ed90d2eb35d */\ +/* sqr(t[0], t[0]); */ /* 161: d0088f51cbff34d258dd3db21a5d66ba */\ +/* sqr(t[0], t[0]); */ /* 162: 1a0111ea397fe69a4b1ba7b6434bacd74 */\ +/* sqr(t[0], t[0]); */ /* 163: 340223d472ffcd3496374f6c869759ae8 */\ +/* sqr(t[0], t[0]); */ /* 164: 680447a8e5ff9a692c6e9ed90d2eb35d0 */\ +sqr_n_mul(t[0], t[0], 4, t[9]); /* 165: 680447a8e5ff9a692c6e9ed90d2eb35d9 */\ +/* sqr(t[0], t[0]); */ /* 166: d0088f51cbff34d258dd3db21a5d66bb2 */\ +/* sqr(t[0], t[0]); */ /* 167: 1a0111ea397fe69a4b1ba7b6434bacd764 */\ +/* sqr(t[0], t[0]); */ /* 168: 340223d472ffcd3496374f6c869759aec8 */\ +/* sqr(t[0], t[0]); */ /* 169: 680447a8e5ff9a692c6e9ed90d2eb35d90 */\ +/* sqr(t[0], t[0]); */ /* 170: d0088f51cbff34d258dd3db21a5d66bb20 */\ +/* sqr(t[0], t[0]); */ /* 171: 1a0111ea397fe69a4b1ba7b6434bacd7640 */\ +/* sqr(t[0], t[0]); */ /* 172: 340223d472ffcd3496374f6c869759aec80 */\ +/* sqr(t[0], t[0]); */ /* 173: 680447a8e5ff9a692c6e9ed90d2eb35d900 */\ +sqr_n_mul(t[0], t[0], 8, t[5]); /* 174: 680447a8e5ff9a692c6e9ed90d2eb35d91d */\ +/* sqr(t[0], t[0]); */ /* 175: d0088f51cbff34d258dd3db21a5d66bb23a */\ +/* sqr(t[0], t[0]); */ /* 176: 1a0111ea397fe69a4b1ba7b6434bacd76474 */\ +/* sqr(t[0], t[0]); */ /* 177: 340223d472ffcd3496374f6c869759aec8e8 */\ +/* sqr(t[0], t[0]); */ /* 178: 680447a8e5ff9a692c6e9ed90d2eb35d91d0 */\ +sqr_n_mul(t[0], t[0], 4, t[3]); /* 179: 680447a8e5ff9a692c6e9ed90d2eb35d91dd */\ +/* sqr(t[0], t[0]); */ /* 180: d0088f51cbff34d258dd3db21a5d66bb23ba */\ +/* sqr(t[0], t[0]); */ /* 181: 1a0111ea397fe69a4b1ba7b6434bacd764774 */\ +/* sqr(t[0], t[0]); */ /* 182: 340223d472ffcd3496374f6c869759aec8ee8 */\ +/* sqr(t[0], t[0]); */ /* 183: 680447a8e5ff9a692c6e9ed90d2eb35d91dd0 */\ +/* sqr(t[0], t[0]); */ /* 184: d0088f51cbff34d258dd3db21a5d66bb23ba0 */\ +/* sqr(t[0], t[0]); */ /* 185: 1a0111ea397fe69a4b1ba7b6434bacd7647740 */\ +/* sqr(t[0], t[0]); */ /* 186: 340223d472ffcd3496374f6c869759aec8ee80 */\ +sqr_n_mul(t[0], t[0], 7, t[11]); /* 187: 340223d472ffcd3496374f6c869759aec8ee97 */\ +/* sqr(t[0], t[0]); */ /* 188: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e */\ +/* sqr(t[0], t[0]); */ /* 189: d0088f51cbff34d258dd3db21a5d66bb23ba5c */\ +/* sqr(t[0], t[0]); */ /* 190: 1a0111ea397fe69a4b1ba7b6434bacd764774b8 */\ +/* sqr(t[0], t[0]); */ /* 191: 340223d472ffcd3496374f6c869759aec8ee970 */\ +/* sqr(t[0], t[0]); */ /* 192: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e0 */\ +/* sqr(t[0], t[0]); */ /* 193: d0088f51cbff34d258dd3db21a5d66bb23ba5c0 */\ +/* sqr(t[0], t[0]); */ /* 194: 1a0111ea397fe69a4b1ba7b6434bacd764774b80 */\ +/* sqr(t[0], t[0]); */ /* 195: 340223d472ffcd3496374f6c869759aec8ee9700 */\ +/* sqr(t[0], t[0]); */ /* 196: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e00 */\ +sqr_n_mul(t[0], t[0], 9, t[10]); /* 197: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13 */\ +/* sqr(t[0], t[0]); */ /* 198: d0088f51cbff34d258dd3db21a5d66bb23ba5c26 */\ +/* sqr(t[0], t[0]); */ /* 199: 1a0111ea397fe69a4b1ba7b6434bacd764774b84c */\ +sqr_n_mul(t[0], t[0], 2, t[8]); /* 200: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f */\ +/* sqr(t[0], t[0]); */ /* 201: 340223d472ffcd3496374f6c869759aec8ee9709e */\ +/* sqr(t[0], t[0]); */ /* 202: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13c */\ +/* sqr(t[0], t[0]); */ /* 203: d0088f51cbff34d258dd3db21a5d66bb23ba5c278 */\ +/* sqr(t[0], t[0]); */ /* 204: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f0 */\ +/* sqr(t[0], t[0]); */ /* 205: 340223d472ffcd3496374f6c869759aec8ee9709e0 */\ +sqr_n_mul(t[0], t[0], 5, t[6]); /* 206: 340223d472ffcd3496374f6c869759aec8ee9709e7 */\ +/* sqr(t[0], t[0]); */ /* 207: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce */\ +/* sqr(t[0], t[0]); */ /* 208: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c */\ +/* sqr(t[0], t[0]); */ /* 209: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38 */\ +/* sqr(t[0], t[0]); */ /* 210: 340223d472ffcd3496374f6c869759aec8ee9709e70 */\ +/* sqr(t[0], t[0]); */ /* 211: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce0 */\ +/* sqr(t[0], t[0]); */ /* 212: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c0 */\ +/* sqr(t[0], t[0]); */ /* 213: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f380 */\ +sqr_n_mul(t[0], t[0], 7, t[1]); /* 214: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f385 */\ +/* sqr(t[0], t[0]); */ /* 215: 340223d472ffcd3496374f6c869759aec8ee9709e70a */\ +/* sqr(t[0], t[0]); */ /* 216: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce14 */\ +/* sqr(t[0], t[0]); */ /* 217: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c28 */\ +/* sqr(t[0], t[0]); */ /* 218: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f3850 */\ +/* sqr(t[0], t[0]); */ /* 219: 340223d472ffcd3496374f6c869759aec8ee9709e70a0 */\ +/* sqr(t[0], t[0]); */ /* 220: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce140 */\ +/* sqr(t[0], t[0]); */ /* 221: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c280 */\ +sqr_n_mul(t[0], t[0], 7, t[9]); /* 222: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c289 */\ +/* sqr(t[0], t[0]); */ /* 223: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512 */\ +/* sqr(t[0], t[0]); */ /* 224: 340223d472ffcd3496374f6c869759aec8ee9709e70a24 */\ +/* sqr(t[0], t[0]); */ /* 225: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce1448 */\ +/* sqr(t[0], t[0]); */ /* 226: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2890 */\ +/* sqr(t[0], t[0]); */ /* 227: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f385120 */\ +/* sqr(t[0], t[0]); */ /* 228: 340223d472ffcd3496374f6c869759aec8ee9709e70a240 */\ +sqr_n_mul(t[0], t[0], 6, t[11]); /* 229: 340223d472ffcd3496374f6c869759aec8ee9709e70a257 */\ +/* sqr(t[0], t[0]); */ /* 230: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144ae */\ +/* sqr(t[0], t[0]); */ /* 231: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895c */\ +/* sqr(t[0], t[0]); */ /* 232: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512b8 */\ +/* sqr(t[0], t[0]); */ /* 233: 340223d472ffcd3496374f6c869759aec8ee9709e70a2570 */\ +/* sqr(t[0], t[0]); */ /* 234: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144ae0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 235: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd */\ +/* sqr(t[0], t[0]); */ /* 236: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fa */\ +/* sqr(t[0], t[0]); */ /* 237: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf4 */\ +/* sqr(t[0], t[0]); */ /* 238: 340223d472ffcd3496374f6c869759aec8ee9709e70a257e8 */\ +/* sqr(t[0], t[0]); */ /* 239: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd0 */\ +/* sqr(t[0], t[0]); */ /* 240: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fa0 */\ +sqr_n_mul(t[0], t[0], 5, t[10]); /* 241: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3 */\ +/* sqr(t[0], t[0]); */ /* 242: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf66 */\ +/* sqr(t[0], t[0]); */ /* 243: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ecc */\ +/* sqr(t[0], t[0]); */ /* 244: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd98 */\ +/* sqr(t[0], t[0]); */ /* 245: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb30 */\ +/* sqr(t[0], t[0]); */ /* 246: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf660 */\ +sqr_n_mul(t[0], t[0], 5, t[10]); /* 247: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf673 */\ +/* sqr(t[0], t[0]); */ /* 248: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece6 */\ +/* sqr(t[0], t[0]); */ /* 249: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc */\ +/* sqr(t[0], t[0]); */ /* 250: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398 */\ +/* sqr(t[0], t[0]); */ /* 251: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730 */\ +/* sqr(t[0], t[0]); */ /* 252: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece60 */\ +/* sqr(t[0], t[0]); */ /* 253: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc0 */\ +/* sqr(t[0], t[0]); */ /* 254: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3980 */\ +/* sqr(t[0], t[0]); */ /* 255: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf67300 */\ +sqr_n_mul(t[0], t[0], 8, t[3]); /* 256: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d */\ +/* sqr(t[0], t[0]); */ /* 257: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a */\ +/* sqr(t[0], t[0]); */ /* 258: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34 */\ +/* sqr(t[0], t[0]); */ /* 259: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39868 */\ +/* sqr(t[0], t[0]); */ /* 260: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d0 */\ +/* sqr(t[0], t[0]); */ /* 261: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a0 */\ +/* sqr(t[0], t[0]); */ /* 262: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc340 */\ +/* sqr(t[0], t[0]); */ /* 263: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398680 */\ +sqr_n_mul(t[0], t[0], 7, t[2]); /* 264: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398695 */\ +/* sqr(t[0], t[0]); */ /* 265: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a */\ +/* sqr(t[0], t[0]); */ /* 266: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a54 */\ +/* sqr(t[0], t[0]); */ /* 267: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a8 */\ +/* sqr(t[0], t[0]); */ /* 268: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb3986950 */\ +/* sqr(t[0], t[0]); */ /* 269: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0 */\ +/* sqr(t[0], t[0]); */ /* 270: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a540 */\ +/* sqr(t[0], t[0]); */ /* 271: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a80 */\ +/* sqr(t[0], t[0]); */ /* 272: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869500 */\ +/* sqr(t[0], t[0]); */ /* 273: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a00 */\ +sqr_n_mul(t[0], t[0], 9, t[7]); /* 274: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f */\ +/* sqr(t[0], t[0]); */ /* 275: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541e */\ +/* sqr(t[0], t[0]); */ /* 276: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83c */\ +/* sqr(t[0], t[0]); */ /* 277: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb398695078 */\ +/* sqr(t[0], t[0]); */ /* 278: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f0 */\ +/* sqr(t[0], t[0]); */ /* 279: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541e0 */\ +sqr_n_mul(t[0], t[0], 5, t[3]); /* 280: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed */\ +/* sqr(t[0], t[0]); */ /* 281: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83da */\ +/* sqr(t[0], t[0]); */ /* 282: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b4 */\ +/* sqr(t[0], t[0]); */ /* 283: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f68 */\ +sqr_n_mul(t[0], t[0], 3, t[8]); /* 284: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b */\ +/* sqr(t[0], t[0]); */ /* 285: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed6 */\ +/* sqr(t[0], t[0]); */ /* 286: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac */\ +/* sqr(t[0], t[0]); */ /* 287: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b58 */\ +/* sqr(t[0], t[0]); */ /* 288: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0 */\ +/* sqr(t[0], t[0]); */ /* 289: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed60 */\ +/* sqr(t[0], t[0]); */ /* 290: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac0 */\ +/* sqr(t[0], t[0]); */ /* 291: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b580 */\ +/* sqr(t[0], t[0]); */ /* 292: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b00 */\ +sqr_n_mul(t[0], t[0], 8, t[7]); /* 293: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f */\ +/* sqr(t[0], t[0]); */ /* 294: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61e */\ +/* sqr(t[0], t[0]); */ /* 295: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3c */\ +/* sqr(t[0], t[0]); */ /* 296: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b5878 */\ +sqr_n_mul(t[0], t[0], 3, t[8]); /* 297: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b */\ +/* sqr(t[0], t[0]); */ /* 298: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6 */\ +/* sqr(t[0], t[0]); */ /* 299: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec */\ +/* sqr(t[0], t[0]); */ /* 300: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8 */\ +/* sqr(t[0], t[0]); */ /* 301: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b0 */\ +/* sqr(t[0], t[0]); */ /* 302: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f60 */\ +/* sqr(t[0], t[0]); */ /* 303: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec0 */\ +/* sqr(t[0], t[0]); */ /* 304: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d80 */\ +sqr_n_mul(t[0], t[0], 7, t[9]); /* 305: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d89 */\ +/* sqr(t[0], t[0]); */ /* 306: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b12 */\ +/* sqr(t[0], t[0]); */ /* 307: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f624 */\ +/* sqr(t[0], t[0]); */ /* 308: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec48 */\ +/* sqr(t[0], t[0]); */ /* 309: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d890 */\ +/* sqr(t[0], t[0]); */ /* 310: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120 */\ +/* sqr(t[0], t[0]); */ /* 311: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6240 */\ +/* sqr(t[0], t[0]); */ /* 312: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec480 */\ +/* sqr(t[0], t[0]); */ /* 313: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8900 */\ +/* sqr(t[0], t[0]); */ /* 314: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b1200 */\ +sqr_n_mul(t[0], t[0], 9, t[7]); /* 315: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f */\ +/* sqr(t[0], t[0]); */ /* 316: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241e */\ +/* sqr(t[0], t[0]); */ /* 317: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483c */\ +/* sqr(t[0], t[0]); */ /* 318: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d89078 */\ +/* sqr(t[0], t[0]); */ /* 319: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f0 */\ +/* sqr(t[0], t[0]); */ /* 320: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241e0 */\ +/* sqr(t[0], t[0]); */ /* 321: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483c0 */\ +sqr_n_mul(t[0], t[0], 6, t[2]); /* 322: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d5 */\ +/* sqr(t[0], t[0]); */ /* 323: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aa */\ +/* sqr(t[0], t[0]); */ /* 324: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f54 */\ +/* sqr(t[0], t[0]); */ /* 325: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241ea8 */\ +/* sqr(t[0], t[0]); */ /* 326: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d50 */\ +/* sqr(t[0], t[0]); */ /* 327: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aa0 */\ +/* sqr(t[0], t[0]); */ /* 328: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f540 */\ +sqr_n_mul(t[0], t[0], 6, t[4]); /* 329: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55f */\ +/* sqr(t[0], t[0]); */ /* 330: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabe */\ +/* sqr(t[0], t[0]); */ /* 331: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57c */\ +/* sqr(t[0], t[0]); */ /* 332: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaf8 */\ +/* sqr(t[0], t[0]); */ /* 333: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55f0 */\ +/* sqr(t[0], t[0]); */ /* 334: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 335: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabff */\ +/* sqr(t[0], t[0]); */ /* 336: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fe */\ +/* sqr(t[0], t[0]); */ /* 337: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffc */\ +/* sqr(t[0], t[0]); */ /* 338: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ff8 */\ +/* sqr(t[0], t[0]); */ /* 339: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabff0 */\ +/* sqr(t[0], t[0]); */ /* 340: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 341: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fff */\ +/* sqr(t[0], t[0]); */ /* 342: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aafffe */\ +/* sqr(t[0], t[0]); */ /* 343: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55fffc */\ +/* sqr(t[0], t[0]); */ /* 344: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfff8 */\ +/* sqr(t[0], t[0]); */ /* 345: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fff0 */\ +sqr_n_mul(t[0], t[0], 4, t[3]); /* 346: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd */\ +/* sqr(t[0], t[0]); */ /* 347: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffa */\ +/* sqr(t[0], t[0]); */ /* 348: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff4 */\ +/* sqr(t[0], t[0]); */ /* 349: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffe8 */\ +sqr_n_mul(t[0], t[0], 3, t[8]); /* 350: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb */\ +/* sqr(t[0], t[0]); */ /* 351: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd6 */\ +/* sqr(t[0], t[0]); */ /* 352: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac */\ +/* sqr(t[0], t[0]); */ /* 353: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58 */\ +/* sqr(t[0], t[0]); */ /* 354: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb0 */\ +/* sqr(t[0], t[0]); */ /* 355: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd60 */\ +/* sqr(t[0], t[0]); */ /* 356: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac0 */\ +/* sqr(t[0], t[0]); */ /* 357: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff580 */\ +/* sqr(t[0], t[0]); */ /* 358: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb00 */\ +sqr_n_mul(t[0], t[0], 8, t[2]); /* 359: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb15 */\ +/* sqr(t[0], t[0]); */ /* 360: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a */\ +/* sqr(t[0], t[0]); */ /* 361: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54 */\ +/* sqr(t[0], t[0]); */ /* 362: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a8 */\ +/* sqr(t[0], t[0]); */ /* 363: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb150 */\ +/* sqr(t[0], t[0]); */ /* 364: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a0 */\ +/* sqr(t[0], t[0]); */ /* 365: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac540 */\ +/* sqr(t[0], t[0]); */ /* 366: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a80 */\ +sqr_n_mul(t[0], t[0], 7, t[4]); /* 367: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9f */\ +/* sqr(t[0], t[0]); */ /* 368: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153e */\ +/* sqr(t[0], t[0]); */ /* 369: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7c */\ +/* sqr(t[0], t[0]); */ /* 370: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54f8 */\ +/* sqr(t[0], t[0]); */ /* 371: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9f0 */\ +/* sqr(t[0], t[0]); */ /* 372: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153e0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 373: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ff */\ +/* sqr(t[0], t[0]); */ /* 374: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fe */\ +/* sqr(t[0], t[0]); */ /* 375: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffc */\ +/* sqr(t[0], t[0]); */ /* 376: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ff8 */\ +/* sqr(t[0], t[0]); */ /* 377: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ff0 */\ +/* sqr(t[0], t[0]); */ /* 378: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 379: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fff */\ +/* sqr(t[0], t[0]); */ /* 380: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54fffe */\ +/* sqr(t[0], t[0]); */ /* 381: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9fffc */\ +/* sqr(t[0], t[0]); */ /* 382: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153fff8 */\ +/* sqr(t[0], t[0]); */ /* 383: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7fff0 */\ +sqr_n_mul(t[0], t[0], 4, t[7]); /* 384: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff */\ +/* sqr(t[0], t[0]); */ /* 385: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffe */\ +/* sqr(t[0], t[0]); */ /* 386: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffc */\ +/* sqr(t[0], t[0]); */ /* 387: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffff8 */\ +/* sqr(t[0], t[0]); */ /* 388: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff0 */\ +sqr_n_mul(t[0], t[0], 4, t[6]); /* 389: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff7 */\ +/* sqr(t[0], t[0]); */ /* 390: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee */\ +/* sqr(t[0], t[0]); */ /* 391: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdc */\ +/* sqr(t[0], t[0]); */ /* 392: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb8 */\ +/* sqr(t[0], t[0]); */ /* 393: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff70 */\ +/* sqr(t[0], t[0]); */ /* 394: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee0 */\ +/* sqr(t[0], t[0]); */ /* 395: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdc0 */\ +/* sqr(t[0], t[0]); */ /* 396: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb80 */\ +sqr_n_mul(t[0], t[0], 7, t[4]); /* 397: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9f */\ +/* sqr(t[0], t[0]); */ /* 398: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73e */\ +/* sqr(t[0], t[0]); */ /* 399: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7c */\ +/* sqr(t[0], t[0]); */ /* 400: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcf8 */\ +/* sqr(t[0], t[0]); */ /* 401: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9f0 */\ +/* sqr(t[0], t[0]); */ /* 402: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73e0 */\ +sqr_n_mul(t[0], t[0], 5, t[5]); /* 403: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fd */\ +/* sqr(t[0], t[0]); */ /* 404: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fa */\ +/* sqr(t[0], t[0]); */ /* 405: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff4 */\ +/* sqr(t[0], t[0]); */ /* 406: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fe8 */\ +/* sqr(t[0], t[0]); */ /* 407: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fd0 */\ +/* sqr(t[0], t[0]); */ /* 408: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fa0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 409: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbf */\ +/* sqr(t[0], t[0]); */ /* 410: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7e */\ +/* sqr(t[0], t[0]); */ /* 411: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefc */\ +/* sqr(t[0], t[0]); */ /* 412: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdf8 */\ +/* sqr(t[0], t[0]); */ /* 413: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbf0 */\ +/* sqr(t[0], t[0]); */ /* 414: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7e0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 415: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ff */\ +/* sqr(t[0], t[0]); */ /* 416: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffe */\ +/* sqr(t[0], t[0]); */ /* 417: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffc */\ +/* sqr(t[0], t[0]); */ /* 418: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbff8 */\ +/* sqr(t[0], t[0]); */ /* 419: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ff0 */\ +/* sqr(t[0], t[0]); */ /* 420: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 421: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffff */\ +/* sqr(t[0], t[0]); */ /* 422: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffe */\ +/* sqr(t[0], t[0]); */ /* 423: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffc */\ +/* sqr(t[0], t[0]); */ /* 424: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fff8 */\ +/* sqr(t[0], t[0]); */ /* 425: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffff0 */\ +/* sqr(t[0], t[0]); */ /* 426: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 427: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffff */\ +/* sqr(t[0], t[0]); */ /* 428: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffe */\ +/* sqr(t[0], t[0]); */ /* 429: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7ffffc */\ +/* sqr(t[0], t[0]); */ /* 430: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefffff8 */\ +/* sqr(t[0], t[0]); */ /* 431: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffff0 */\ +/* sqr(t[0], t[0]); */ /* 432: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 433: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffff */\ +/* sqr(t[0], t[0]); */ /* 434: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffe */\ +/* sqr(t[0], t[0]); */ /* 435: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffc */\ +/* sqr(t[0], t[0]); */ /* 436: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffff8 */\ +/* sqr(t[0], t[0]); */ /* 437: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbffffff0 */\ +/* sqr(t[0], t[0]); */ /* 438: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffe0 */\ +sqr_n_mul(t[0], t[0], 5, t[4]); /* 439: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffff */\ +/* sqr(t[0], t[0]); */ /* 440: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9fefffffffe */\ +/* sqr(t[0], t[0]); */ /* 441: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdfffffffc */\ +/* sqr(t[0], t[0]); */ /* 442: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffff8 */\ +/* sqr(t[0], t[0]); */ /* 443: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffff0 */\ +sqr_n_mul(t[0], t[0], 4, t[3]); /* 444: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd */\ +/* sqr(t[0], t[0]); */ /* 445: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffa */\ +/* sqr(t[0], t[0]); */ /* 446: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff4 */\ +/* sqr(t[0], t[0]); */ /* 447: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffe8 */\ +/* sqr(t[0], t[0]); */ /* 448: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd0 */\ +/* sqr(t[0], t[0]); */ /* 449: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffa0 */\ +/* sqr(t[0], t[0]); */ /* 450: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff40 */\ +sqr_n_mul(t[0], t[0], 6, t[2]); /* 451: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff55 */\ +/* sqr(t[0], t[0]); */ /* 452: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaa */\ +/* sqr(t[0], t[0]); */ /* 453: d0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd54 */\ +/* sqr(t[0], t[0]); */ /* 454: 1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaa8 */\ +/* sqr(t[0], t[0]); */ /* 455: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff550 */\ +sqr_n_mul(t[0], t[0], 4, t[1]); /* 456: 340223d472ffcd3496374f6c869759aec8ee9709e70a257ece61a541ed61ec483d57fffd62a7ffff73fdffffffff555 */\ +sqr(out, t[0]); /* 457: 680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaaa */\ +} while(0) diff --git a/crypto/blst_src/sqrt.c b/crypto/blst_src/sqrt.c new file mode 100644 index 00000000000..cf149fd1124 --- /dev/null +++ b/crypto/blst_src/sqrt.c @@ -0,0 +1,261 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fields.h" + +#ifdef __OPTIMIZE_SIZE__ +static void recip_sqrt_fp_3mod4(vec384 out, const vec384 inp) +{ + static const byte BLS_12_381_P_minus_3_div_4[] = { + TO_BYTES(0xee7fbfffffffeaaa), TO_BYTES(0x07aaffffac54ffff), + TO_BYTES(0xd9cc34a83dac3d89), TO_BYTES(0xd91dd2e13ce144af), + TO_BYTES(0x92c6e9ed90d2eb35), TO_BYTES(0x0680447a8e5ff9a6) + }; + + exp_mont_384(out, inp, BLS_12_381_P_minus_3_div_4, 379, BLS12_381_P, p0); +} +#else +# if 1 +/* + * "383"-bit variant omits full reductions at the ends of squarings, + * which results in up to ~15% improvement. [One can improve further + * by omitting full reductions even after multiplications and + * performing final reduction at the very end of the chain.] + */ +static inline void sqr_n_mul_fp(vec384 out, const vec384 a, size_t count, + const vec384 b) +{ sqr_n_mul_mont_383(out, a, count, BLS12_381_P, p0, b); } +# else +static void sqr_n_mul_fp(vec384 out, const vec384 a, size_t count, + const vec384 b) +{ + while(count--) { + sqr_fp(out, a); + a = out; + } + mul_fp(out, out, b); +} +# endif + +# define sqr(ret,a) sqr_fp(ret,a) +# define mul(ret,a,b) mul_fp(ret,a,b) +# define sqr_n_mul(ret,a,n,b) sqr_n_mul_fp(ret,a,n,b) + +# include "sqrt-addchain.h" +static void recip_sqrt_fp_3mod4(vec384 out, const vec384 inp) +{ + RECIP_SQRT_MOD_BLS12_381_P(out, inp, vec384); +} +# undef RECIP_SQRT_MOD_BLS12_381_P + +# undef sqr_n_mul +# undef sqr +# undef mul +#endif + +static bool_t recip_sqrt_fp(vec384 out, const vec384 inp) +{ + vec384 t0, t1; + bool_t ret; + + recip_sqrt_fp_3mod4(t0, inp); + + mul_fp(t1, t0, inp); + sqr_fp(t1, t1); + ret = vec_is_equal(t1, inp, sizeof(t1)); + vec_copy(out, t0, sizeof(t0)); + + return ret; +} + +static bool_t sqrt_fp(vec384 out, const vec384 inp) +{ + vec384 t0, t1; + bool_t ret; + + recip_sqrt_fp_3mod4(t0, inp); + + mul_fp(t0, t0, inp); + sqr_fp(t1, t0); + ret = vec_is_equal(t1, inp, sizeof(t1)); + vec_copy(out, t0, sizeof(t0)); + + return ret; +} + +int blst_fp_sqrt(vec384 out, const vec384 inp) +{ return (int)sqrt_fp(out, inp); } + +int blst_fp_is_square(const vec384 inp) +{ + return (int)ct_is_square_mod_384(inp, BLS12_381_P); +} + +static bool_t sqrt_align_fp2(vec384x out, const vec384x ret, + const vec384x sqrt, const vec384x inp) +{ + static const vec384x sqrt_minus_1 = { { 0 }, { ONE_MONT_P } }; + static const vec384x sqrt_sqrt_minus_1 = { + /* + * "magic" number is ±2^((p-3)/4)%p, which is "1/sqrt(2)", + * in quotes because 2*"1/sqrt(2)"^2 == -1 mod p, not 1, + * but it pivots into "complex" plane nevertheless... + */ + { TO_LIMB_T(0x3e2f585da55c9ad1), TO_LIMB_T(0x4294213d86c18183), + TO_LIMB_T(0x382844c88b623732), TO_LIMB_T(0x92ad2afd19103e18), + TO_LIMB_T(0x1d794e4fac7cf0b9), TO_LIMB_T(0x0bd592fc7d825ec8) }, + { TO_LIMB_T(0x7bcfa7a25aa30fda), TO_LIMB_T(0xdc17dec12a927e7c), + TO_LIMB_T(0x2f088dd86b4ebef1), TO_LIMB_T(0xd1ca2087da74d4a7), + TO_LIMB_T(0x2da2596696cebc1d), TO_LIMB_T(0x0e2b7eedbbfd87d2) } + }; + static const vec384x sqrt_minus_sqrt_minus_1 = { + { TO_LIMB_T(0x7bcfa7a25aa30fda), TO_LIMB_T(0xdc17dec12a927e7c), + TO_LIMB_T(0x2f088dd86b4ebef1), TO_LIMB_T(0xd1ca2087da74d4a7), + TO_LIMB_T(0x2da2596696cebc1d), TO_LIMB_T(0x0e2b7eedbbfd87d2) }, + { TO_LIMB_T(0x7bcfa7a25aa30fda), TO_LIMB_T(0xdc17dec12a927e7c), + TO_LIMB_T(0x2f088dd86b4ebef1), TO_LIMB_T(0xd1ca2087da74d4a7), + TO_LIMB_T(0x2da2596696cebc1d), TO_LIMB_T(0x0e2b7eedbbfd87d2) } + }; + vec384x coeff, t0, t1; + bool_t is_sqrt, flag; + + /* + * Instead of multiple trial squarings we can perform just one + * and see if the result is "rotated by multiple of 90°" in + * relation to |inp|, and "rotate" |ret| accordingly. + */ + sqr_fp2(t0, sqrt); + /* "sqrt(|inp|)"^2 = (a + b*i)^2 = (a^2-b^2) + 2ab*i */ + + /* (a^2-b^2) + 2ab*i == |inp| ? |ret| is spot on */ + sub_fp2(t1, t0, inp); + is_sqrt = vec_is_zero(t1, sizeof(t1)); + vec_copy(coeff, BLS12_381_Rx.p2, sizeof(coeff)); + + /* -(a^2-b^2) - 2ab*i == |inp| ? "rotate |ret| by 90°" */ + add_fp2(t1, t0, inp); + vec_select(coeff, sqrt_minus_1, coeff, sizeof(coeff), + flag = vec_is_zero(t1, sizeof(t1))); + is_sqrt |= flag; + + /* 2ab - (a^2-b^2)*i == |inp| ? "rotate |ret| by 135°" */ + sub_fp(t1[0], t0[0], inp[1]); + add_fp(t1[1], t0[1], inp[0]); + vec_select(coeff, sqrt_sqrt_minus_1, coeff, sizeof(coeff), + flag = vec_is_zero(t1, sizeof(t1))); + is_sqrt |= flag; + + /* -2ab + (a^2-b^2)*i == |inp| ? "rotate |ret| by 45°" */ + add_fp(t1[0], t0[0], inp[1]); + sub_fp(t1[1], t0[1], inp[0]); + vec_select(coeff, sqrt_minus_sqrt_minus_1, coeff, sizeof(coeff), + flag = vec_is_zero(t1, sizeof(t1))); + is_sqrt |= flag; + + /* actual "rotation" */ + mul_fp2(out, ret, coeff); + + return is_sqrt; +} + +/* + * |inp| = a + b*i + */ +static bool_t recip_sqrt_fp2(vec384x out, const vec384x inp, + const vec384x recip_ZZZ, + const vec384x magic_ZZZ) +{ + vec384 aa, bb, cc; + vec384x inp_; + bool_t is_sqrt; + + sqr_fp(aa, inp[0]); + sqr_fp(bb, inp[1]); + add_fp(aa, aa, bb); + + is_sqrt = recip_sqrt_fp(cc, aa); /* 1/sqrt(a²+b²) */ + + /* if |inp| doesn't have quadratic residue, multiply by "1/Z³" ... */ + mul_fp2(inp_, inp, recip_ZZZ); + /* ... and adjust |aa| and |cc| accordingly */ + { + vec384 za, zc; + + mul_fp(za, aa, magic_ZZZ[0]); /* aa*(za² + zb²) */ + mul_fp(zc, cc, magic_ZZZ[1]); /* cc*(za² + zb²)^((p-3)/4) */ + vec_select(aa, aa, za, sizeof(aa), is_sqrt); + vec_select(cc, cc, zc, sizeof(cc), is_sqrt); + } + vec_select(inp_, inp, inp_, sizeof(inp_), is_sqrt); + + mul_fp(aa, aa, cc); /* sqrt(a²+b²) */ + + sub_fp(bb, inp_[0], aa); + add_fp(aa, inp_[0], aa); + vec_select(aa, bb, aa, sizeof(aa), vec_is_zero(aa, sizeof(aa))); + div_by_2_fp(aa, aa); /* (a ± sqrt(a²+b²))/2 */ + + /* if it says "no sqrt," final "align" will find right one... */ + (void)recip_sqrt_fp(out[0], aa); /* 1/sqrt((a ± sqrt(a²+b²))/2) */ + + div_by_2_fp(out[1], inp_[1]); + mul_fp(out[1], out[1], out[0]); /* b/(2*sqrt((a ± sqrt(a²+b²))/2)) */ + mul_fp(out[0], out[0], aa); /* sqrt((a ± sqrt(a²+b²))/2) */ + + /* bound to succeed */ + (void)sqrt_align_fp2(out, out, out, inp_); + + mul_fp(out[0], out[0], cc); /* inverse the result */ + mul_fp(out[1], out[1], cc); + neg_fp(out[1], out[1]); + + return is_sqrt; +} + +static bool_t sqrt_fp2(vec384x out, const vec384x inp) +{ + vec384x ret; + vec384 aa, bb; + + sqr_fp(aa, inp[0]); + sqr_fp(bb, inp[1]); + add_fp(aa, aa, bb); + + /* don't pay attention to return value, final "align" will tell... */ + (void)sqrt_fp(aa, aa); /* sqrt(a²+b²) */ + + sub_fp(bb, inp[0], aa); + add_fp(aa, inp[0], aa); + vec_select(aa, bb, aa, sizeof(aa), vec_is_zero(aa, sizeof(aa))); + div_by_2_fp(aa, aa); /* (a ± sqrt(a²+b²))/2 */ + + /* if it says "no sqrt," final "align" will find right one... */ + (void)recip_sqrt_fp(ret[0], aa); /* 1/sqrt((a ± sqrt(a²+b²))/2) */ + + div_by_2_fp(ret[1], inp[1]); + mul_fp(ret[1], ret[1], ret[0]); /* b/(2*sqrt((a ± sqrt(a²+b²))/2)) */ + mul_fp(ret[0], ret[0], aa); /* sqrt((a ± sqrt(a²+b²))/2) */ + + /* + * Now see if |ret| is or can be made sqrt(|inp|)... + */ + + return sqrt_align_fp2(out, ret, ret, inp); +} + +int blst_fp2_sqrt(vec384x out, const vec384x inp) +{ return (int)sqrt_fp2(out, inp); } + +int blst_fp2_is_square(const vec384x inp) +{ + vec384 aa, bb; + + sqr_fp(aa, inp[0]); + sqr_fp(bb, inp[1]); + add_fp(aa, aa, bb); + + return (int)ct_is_square_mod_384(aa, BLS12_381_P); +} diff --git a/crypto/blst_src/vect.c b/crypto/blst_src/vect.c new file mode 100644 index 00000000000..1834a48fadd --- /dev/null +++ b/crypto/blst_src/vect.c @@ -0,0 +1,176 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "vect.h" + +#ifdef __BLST_NO_ASM__ +# include "no_asm.h" +#endif + +/* + * Following are some reference C implementations to assist new + * assembly modules development, as starting-point stand-ins and for + * cross-checking. In order to "polyfil" specific subroutine redefine + * it on compiler command line, e.g. -Dmul_mont_384x=_mul_mont_384x. + */ + +#ifdef lshift_mod_384 +inline void lshift_mod_384(vec384 ret, const vec384 a, size_t n, + const vec384 mod) +{ + while(n--) + add_mod_384(ret, a, a, mod), a = ret; +} +#endif + +#ifdef mul_by_8_mod_384 +inline void mul_by_8_mod_384(vec384 ret, const vec384 a, const vec384 mod) +{ lshift_mod_384(ret, a, 3, mod); } +#endif + +#ifdef mul_by_3_mod_384 +inline void mul_by_3_mod_384(vec384 ret, const vec384 a, const vec384 mod) +{ + vec384 t; + + add_mod_384(t, a, a, mod); + add_mod_384(ret, t, a, mod); +} +#endif + +#ifdef mul_by_3_mod_384x +inline void mul_by_3_mod_384x(vec384x ret, const vec384x a, const vec384 mod) +{ + mul_by_3_mod_384(ret[0], a[0], mod); + mul_by_3_mod_384(ret[1], a[1], mod); +} +#endif + +#ifdef mul_by_8_mod_384x +inline void mul_by_8_mod_384x(vec384x ret, const vec384x a, const vec384 mod) +{ + mul_by_8_mod_384(ret[0], a[0], mod); + mul_by_8_mod_384(ret[1], a[1], mod); +} +#endif + +#ifdef mul_by_1_plus_i_mod_384x +inline void mul_by_1_plus_i_mod_384x(vec384x ret, const vec384x a, + const vec384 mod) +{ + vec384 t; + + add_mod_384(t, a[0], a[1], mod); + sub_mod_384(ret[0], a[0], a[1], mod); + vec_copy(ret[1], t, sizeof(t)); +} +#endif + +#ifdef add_mod_384x +inline void add_mod_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 mod) +{ + add_mod_384(ret[0], a[0], b[0], mod); + add_mod_384(ret[1], a[1], b[1], mod); +} +#endif + +#ifdef sub_mod_384x +inline void sub_mod_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 mod) +{ + sub_mod_384(ret[0], a[0], b[0], mod); + sub_mod_384(ret[1], a[1], b[1], mod); +} +#endif + +#ifdef lshift_mod_384x +inline void lshift_mod_384x(vec384x ret, const vec384x a, size_t n, + const vec384 mod) +{ + lshift_mod_384(ret[0], a[0], n, mod); + lshift_mod_384(ret[1], a[1], n, mod); +} +#endif + +#if defined(mul_mont_384x) && !(defined(__ADX__) && !defined(__BLST_PORTABLE__)) +void mul_mont_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 mod, limb_t n0) +{ + vec768 t0, t1, t2; + vec384 aa, bb; + + mul_384(t0, a[0], b[0]); + mul_384(t1, a[1], b[1]); + + add_mod_384(aa, a[0], a[1], mod); + add_mod_384(bb, b[0], b[1], mod); + mul_384(t2, aa, bb); + sub_mod_384x384(t2, t2, t0, mod); + sub_mod_384x384(t2, t2, t1, mod); + + sub_mod_384x384(t0, t0, t1, mod); + + redc_mont_384(ret[0], t0, mod, n0); + redc_mont_384(ret[1], t2, mod, n0); +} +#endif + +#if defined(sqr_mont_384x) && !(defined(__ADX__) && !defined(__BLST_PORTABLE__)) +void sqr_mont_384x(vec384x ret, const vec384x a, const vec384 mod, limb_t n0) +{ + vec384 t0, t1; + + add_mod_384(t0, a[0], a[1], mod); + sub_mod_384(t1, a[0], a[1], mod); + + mul_mont_384(ret[1], a[0], a[1], mod, n0); + add_mod_384(ret[1], ret[1], ret[1], mod); + + mul_mont_384(ret[0], t0, t1, mod, n0); +} +#endif + +limb_t div_3_limbs(const limb_t dividend_top[2], limb_t d_lo, limb_t d_hi); +limb_t quot_rem_128(limb_t *quot_rem, const limb_t *divisor, limb_t quotient); +limb_t quot_rem_64(limb_t *quot_rem, const limb_t *divisor, limb_t quotient); + +/* + * Divide 255-bit |val| by z^2 yielding 128-bit quotient and remainder in place. + */ +static void div_by_zz(limb_t val[]) +{ + static const limb_t zz[] = { TO_LIMB_T(0x0000000100000000), + TO_LIMB_T(0xac45a4010001a402) }; + size_t loop, zz_len = sizeof(zz)/sizeof(zz[0]); + limb_t d_lo, d_hi; + + d_lo = zz[zz_len - 2]; + d_hi = zz[zz_len - 1]; + for (loop = zz_len, zz_len--; loop--;) { + limb_t q = div_3_limbs(val + loop + zz_len, d_lo, d_hi); + (void)quot_rem_128(val + loop, zz, q); + } + /* remainder is in low half of val[], quotient is in high */ +} + +/* + * Divide 128-bit |val| by z yielding 64-bit quotient and remainder in place. + */ +static void div_by_z(limb_t val[]) +{ + static const limb_t z[] = { TO_LIMB_T(0xd201000000010000) }; + size_t loop, z_len = sizeof(z)/sizeof(z[0]); + limb_t d_lo, d_hi; + + d_lo = (sizeof(z) == sizeof(limb_t)) ? 0 : z[z_len - 2]; + d_hi = z[z_len - 1]; + for (loop = z_len, z_len--; loop--;) { + limb_t q = div_3_limbs(val + loop + z_len, d_lo, d_hi); + (void)quot_rem_64(val + loop, z, q); + } + /* remainder is in low half of val[], quotient is in high */ +} diff --git a/crypto/blst_src/vect.h b/crypto/blst_src/vect.h new file mode 100644 index 00000000000..3211c8628cf --- /dev/null +++ b/crypto/blst_src/vect.h @@ -0,0 +1,418 @@ +/* + * Copyright Supranational LLC + * Licensed under the Apache License, Version 2.0, see LICENSE for details. + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef __BLS12_381_ASM_VECT_H__ +#define __BLS12_381_ASM_VECT_H__ + +#include <stddef.h> + +#if defined(__x86_64__) || defined(__aarch64__) +/* These are available even in ILP32 flavours, but even then they are + * capable of performing 64-bit operations as efficiently as in *P64. */ +typedef unsigned long long limb_t; +# define LIMB_T_BITS 64 + +#elif defined(_WIN64) /* Win64 is P64 */ +typedef unsigned __int64 limb_t; +# define LIMB_T_BITS 64 + +#elif defined(__BLST_NO_ASM__) || defined(__wasm64__) +typedef unsigned int limb_t; +# define LIMB_T_BITS 32 +# ifndef __BLST_NO_ASM__ +# define __BLST_NO_ASM__ +# endif + +#else /* 32 bits on 32-bit platforms, 64 - on 64-bit */ +typedef unsigned long limb_t; +# ifdef _LP64 +# define LIMB_T_BITS 64 +# else +# define LIMB_T_BITS 32 +# define __BLST_NO_ASM__ +# endif +#endif + +/* + * Why isn't LIMB_T_BITS defined as 8*sizeof(limb_t)? Because pre-processor + * knows nothing about sizeof(anything)... + */ +#if LIMB_T_BITS == 64 +# define TO_LIMB_T(limb64) limb64 +#else +# define TO_LIMB_T(limb64) (limb_t)limb64,(limb_t)(limb64>>32) +#endif + +#define NLIMBS(bits) (bits/LIMB_T_BITS) + +typedef limb_t vec256[NLIMBS(256)]; +typedef limb_t vec512[NLIMBS(512)]; +typedef limb_t vec384[NLIMBS(384)]; +typedef limb_t vec768[NLIMBS(768)]; +typedef vec384 vec384x[2]; /* 0 is "real" part, 1 is "imaginary" */ + +typedef unsigned char byte; +#define TO_BYTES(limb64) (byte)limb64,(byte)(limb64>>8),\ + (byte)(limb64>>16),(byte)(limb64>>24),\ + (byte)(limb64>>32),(byte)(limb64>>40),\ + (byte)(limb64>>48),(byte)(limb64>>56) +typedef byte pow256[256/8]; + +/* + * Internal Boolean type, Bolean by value, hence safe to cast to or + * reinterpret as 'bool'. + */ +typedef limb_t bool_t; + +/* + * Assembly subroutines... + */ +#if defined(__ADX__) /* e.g. -march=broadwell */ && !defined(__BLST_PORTABLE__)\ + && !defined(__BLST_NO_ASM__) +# define mul_mont_sparse_256 mulx_mont_sparse_256 +# define sqr_mont_sparse_256 sqrx_mont_sparse_256 +# define from_mont_256 fromx_mont_256 +# define redc_mont_256 redcx_mont_256 +# define mul_mont_384 mulx_mont_384 +# define sqr_mont_384 sqrx_mont_384 +# define sqr_n_mul_mont_384 sqrx_n_mul_mont_384 +# define sqr_n_mul_mont_383 sqrx_n_mul_mont_383 +# define mul_384 mulx_384 +# define sqr_384 sqrx_384 +# define redc_mont_384 redcx_mont_384 +# define from_mont_384 fromx_mont_384 +# define sgn0_pty_mont_384 sgn0x_pty_mont_384 +# define sgn0_pty_mont_384x sgn0x_pty_mont_384x +# define ct_inverse_mod_383 ctx_inverse_mod_383 +#elif defined(__BLST_NO_ASM__) +# define ct_inverse_mod_383 ct_inverse_mod_384 +#endif + +void mul_mont_sparse_256(vec256 ret, const vec256 a, const vec256 b, + const vec256 p, limb_t n0); +void sqr_mont_sparse_256(vec256 ret, const vec256 a, const vec256 p, limb_t n0); +void redc_mont_256(vec256 ret, const vec512 a, const vec256 p, limb_t n0); +void from_mont_256(vec256 ret, const vec256 a, const vec256 p, limb_t n0); + +void add_mod_256(vec256 ret, const vec256 a, const vec256 b, const vec256 p); +void sub_mod_256(vec256 ret, const vec256 a, const vec256 b, const vec256 p); +void mul_by_3_mod_256(vec256 ret, const vec256 a, const vec256 p); +void cneg_mod_256(vec256 ret, const vec256 a, bool_t flag, const vec256 p); +void lshift_mod_256(vec256 ret, const vec256 a, size_t count, const vec256 p); +void rshift_mod_256(vec256 ret, const vec256 a, size_t count, const vec256 p); +bool_t eucl_inverse_mod_256(vec256 ret, const vec256 a, const vec256 p, + const vec256 one); +limb_t check_mod_256(const pow256 a, const vec256 p); +limb_t add_n_check_mod_256(pow256 ret, const pow256 a, const pow256 b, + const vec256 p); +limb_t sub_n_check_mod_256(pow256 ret, const pow256 a, const pow256 b, + const vec256 p); + +void vec_prefetch(const void *ptr, size_t len); + +void mul_mont_384(vec384 ret, const vec384 a, const vec384 b, + const vec384 p, limb_t n0); +void sqr_mont_384(vec384 ret, const vec384 a, const vec384 p, limb_t n0); +void sqr_n_mul_mont_384(vec384 ret, const vec384 a, size_t count, + const vec384 p, limb_t n0, const vec384 b); +void sqr_n_mul_mont_383(vec384 ret, const vec384 a, size_t count, + const vec384 p, limb_t n0, const vec384 b); + +void mul_384(vec768 ret, const vec384 a, const vec384 b); +void sqr_384(vec768 ret, const vec384 a); +void redc_mont_384(vec384 ret, const vec768 a, const vec384 p, limb_t n0); +void from_mont_384(vec384 ret, const vec384 a, const vec384 p, limb_t n0); +limb_t sgn0_pty_mont_384(const vec384 a, const vec384 p, limb_t n0); +limb_t sgn0_pty_mont_384x(const vec384x a, const vec384 p, limb_t n0); +limb_t sgn0_pty_mod_384(const vec384 a, const vec384 p); +limb_t sgn0_pty_mod_384x(const vec384x a, const vec384 p); + +void add_mod_384(vec384 ret, const vec384 a, const vec384 b, const vec384 p); +void sub_mod_384(vec384 ret, const vec384 a, const vec384 b, const vec384 p); +void mul_by_8_mod_384(vec384 ret, const vec384 a, const vec384 p); +void mul_by_3_mod_384(vec384 ret, const vec384 a, const vec384 p); +void cneg_mod_384(vec384 ret, const vec384 a, bool_t flag, const vec384 p); +void lshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); +void rshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); +void div_by_2_mod_384(vec384 ret, const vec384 a, const vec384 p); +void ct_inverse_mod_383(vec768 ret, const vec384 inp, const vec384 mod, + const vec384 modx); +void ct_inverse_mod_256(vec512 ret, const vec256 inp, const vec256 mod, + const vec256 modx); +bool_t ct_is_square_mod_384(const vec384 inp, const vec384 mod); + +#if defined(__ADX__) /* e.g. -march=broadwell */ && !defined(__BLST_PORTABLE__) +# define mul_mont_384x mulx_mont_384x +# define sqr_mont_384x sqrx_mont_384x +# define sqr_mont_382x sqrx_mont_382x +# define sqr_n_mul_mont_384x sqrx_n_mul_mont_384x +# define mul_382x mulx_382x +# define sqr_382x sqrx_382x +#endif + +void mul_mont_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 p, limb_t n0); +void sqr_mont_384x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); +void sqr_mont_382x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); +void sqr_n_mul_mont_384x(vec384x ret, const vec384x a, size_t count, + const vec384 p, limb_t n0, const vec384x b); +void mul_382x(vec768 ret[2], const vec384x a, const vec384x b, const vec384 p); +void sqr_382x(vec768 ret[2], const vec384x a, const vec384 p); + +void add_mod_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 p); +void sub_mod_384x(vec384x ret, const vec384x a, const vec384x b, + const vec384 p); +void mul_by_8_mod_384x(vec384x ret, const vec384x a, const vec384 p); +void mul_by_3_mod_384x(vec384x ret, const vec384x a, const vec384 p); +void mul_by_1_plus_i_mod_384x(vec384x ret, const vec384x a, const vec384 p); +void add_mod_384x384(vec768 ret, const vec768 a, const vec768 b, + const vec384 p); +void sub_mod_384x384(vec768 ret, const vec768 a, const vec768 b, + const vec384 p); + +/* + * C subroutines + */ +static void exp_mont_384(vec384 out, const vec384 inp, const byte *pow, + size_t pow_bits, const vec384 p, limb_t n0); +static void exp_mont_384x(vec384x out, const vec384x inp, const byte *pow, + size_t pow_bits, const vec384 p, limb_t n0); +static void div_by_zz(limb_t val[]); +static void div_by_z(limb_t val[]); + +#ifdef __UINTPTR_TYPE__ +typedef __UINTPTR_TYPE__ uptr_t; +#else +typedef const void *uptr_t; +#endif + +#if !defined(restrict) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 +# if defined(__GNUC__) && __GNUC__>=2 +# define restrict __restrict__ +# elif defined(_MSC_VER) +# define restrict __restrict +# else +# define restrict +# endif +# endif +#endif + +#if !defined(inline) && !defined(__cplusplus) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901 +# if defined(__GNUC__) && __GNUC__>=2 +# define inline __inline__ +# elif defined(_MSC_VER) +# define inline __inline +# else +# define inline +# endif +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define launder(var) asm volatile("" : "+r"(var)) +#else +# define launder(var) +#endif + +static inline bool_t is_bit_set(const byte *v, size_t i) +{ + bool_t ret = (v[i/8] >> (i%8)) & 1; + launder(ret); + return ret; +} + +static inline bool_t byte_is_zero(unsigned char c) +{ + limb_t ret = ((limb_t)(c) - 1) >> (LIMB_T_BITS - 1); + launder(ret); + return ret; +} + +static inline bool_t bytes_are_zero(const unsigned char *a, size_t num) +{ + unsigned char acc; + size_t i; + + for (acc = 0, i = 0; i < num; i++) + acc |= a[i]; + + return byte_is_zero(acc); +} + +static inline void vec_cswap(void *restrict a, void *restrict b, size_t num, + bool_t cbit) +{ + limb_t ai, *ap = (limb_t *)a; + limb_t bi, *bp = (limb_t *)b; + limb_t xorm, mask = (limb_t)0 - cbit; + size_t i; + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) { + xorm = ((ai = ap[i]) ^ (bi = bp[i])) & mask; + ap[i] = ai ^ xorm; + bp[i] = bi ^ xorm; + } +} + +/* ret = bit ? a : b */ +void vec_select_32(void *ret, const void *a, const void *b, bool_t sel_a); +void vec_select_48(void *ret, const void *a, const void *b, bool_t sel_a); +void vec_select_96(void *ret, const void *a, const void *b, bool_t sel_a); +void vec_select_144(void *ret, const void *a, const void *b, bool_t sel_a); +void vec_select_192(void *ret, const void *a, const void *b, bool_t sel_a); +void vec_select_288(void *ret, const void *a, const void *b, bool_t sel_a); +static inline void vec_select(void *ret, const void *a, const void *b, + size_t num, bool_t sel_a) +{ + launder(sel_a); +#ifndef __BLST_NO_ASM__ + if (num == 32) vec_select_32(ret, a, b, sel_a); + else if (num == 48) vec_select_48(ret, a, b, sel_a); + else if (num == 96) vec_select_96(ret, a, b, sel_a); + else if (num == 144) vec_select_144(ret, a, b, sel_a); + else if (num == 192) vec_select_192(ret, a, b, sel_a); + else if (num == 288) vec_select_288(ret, a, b, sel_a); +#else + if (0) ; +#endif + else { + limb_t bi; + volatile limb_t *rp = (limb_t *)ret; + const limb_t *ap = (const limb_t *)a; + const limb_t *bp = (const limb_t *)b; + limb_t xorm, mask = (limb_t)0 - sel_a; + size_t i; + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) { + xorm = (ap[i] ^ (bi = bp[i])) & mask; + rp[i] = bi ^ xorm; + } + } +} + +static inline bool_t is_zero(limb_t l) +{ + limb_t ret = (~l & (l - 1)) >> (LIMB_T_BITS - 1); + launder(ret); + return ret; +} + +static inline bool_t vec_is_zero(const void *a, size_t num) +{ + const limb_t *ap = (const limb_t *)a; + limb_t acc; + size_t i; + +#ifndef __BLST_NO_ASM__ + bool_t vec_is_zero_16x(const void *a, size_t num); + if ((num & 15) == 0) + return vec_is_zero_16x(a, num); +#endif + + num /= sizeof(limb_t); + + for (acc = 0, i = 0; i < num; i++) + acc |= ap[i]; + + return is_zero(acc); +} + +static inline bool_t vec_is_equal(const void *a, const void *b, size_t num) +{ + const limb_t *ap = (const limb_t *)a; + const limb_t *bp = (const limb_t *)b; + limb_t acc; + size_t i; + +#ifndef __BLST_NO_ASM__ + bool_t vec_is_equal_16x(const void *a, const void *b, size_t num); + if ((num & 15) == 0) + return vec_is_equal_16x(a, b, num); +#endif + + num /= sizeof(limb_t); + + for (acc = 0, i = 0; i < num; i++) + acc |= ap[i] ^ bp[i]; + + return is_zero(acc); +} + +static inline void cneg_mod_384x(vec384x ret, const vec384x a, bool_t flag, + const vec384 p) +{ + cneg_mod_384(ret[0], a[0], flag, p); + cneg_mod_384(ret[1], a[1], flag, p); +} + +static inline void vec_copy(void *restrict ret, const void *a, size_t num) +{ + limb_t *rp = (limb_t *)ret; + const limb_t *ap = (const limb_t *)a; + size_t i; + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) + rp[i] = ap[i]; +} + +static inline void vec_zero(void *ret, size_t num) +{ + volatile limb_t *rp = (volatile limb_t *)ret; + size_t i; + + num /= sizeof(limb_t); + + for (i = 0; i < num; i++) + rp[i] = 0; + +#if defined(__GNUC__) || defined(__clang__) + asm volatile("" : : "r"(ret) : "memory"); +#endif +} + +/* + * Some compilers get arguably overzealous(*) when passing pointer to + * multi-dimensional array [such as vec384x] as 'const' argument. + * General direction seems to be to legitimize such constification, + * so it's argued that suppressing the warning is appropriate. + * + * (*) http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1923.htm + */ +#if defined(__INTEL_COMPILER) +# pragma warning(disable:167) +# pragma warning(disable:556) +#elif defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic ignored "-Wpedantic" +#elif defined(_MSC_VER) +# pragma warning(disable: 4127 4189) +#endif + +#if !defined(__wasm__) +# include <stdlib.h> +#endif + +#if defined(__GNUC__) +# ifndef alloca +# define alloca(s) __builtin_alloca(s) +# endif +#elif defined(__sun) +# include <alloca.h> +#elif defined(_WIN32) +# include <malloc.h> +# ifndef alloca +# define alloca(s) _alloca(s) +# endif +#endif + +#endif /* __BLS12_381_ASM_VECT_H__ */ diff --git a/crypto/common.go b/crypto/common.go index 125a35c97fb..f476de92e3f 100644 --- a/crypto/common.go +++ b/crypto/common.go @@ -1,6 +1,7 @@ package crypto import ( + "crypto/rand" "errors" "fmt" ) @@ -15,14 +16,29 @@ const ( // This is used as a reference but it doesn't mean all implemented primitives provide this minimum. securityBits = 128 - // Relic internal constant (related to exported constants above) - // max byte length of bn_st set to 2048 bits - maxScalarSize = 256 + // keygen seed length conditions + // enforce seed to be at least double the security bits and have enough entropy. + // it is still recommened that seed is generated using a secure RNG. + KeyGenSeedMinLen = 2 * (securityBits / 8) + KeyGenSeedMaxLen = 256 // max relic PRG seed length in bytes maxRelicPrgSeed = 1 << 32 ) +// TODO: update this code to make sure +// the function isn't removed by the compiler +// https://github.com/golang/go/issues/21865 +func overwrite(data []byte) { + _, err := rand.Read(data) // checking err is enough + if err != nil { + // zero the buffer if randomizing failed + for i := 0; i < len(data); i++ { + data[i] = 0 + } + } +} + // invalidInputsError is an error returned when a crypto API receives invalid inputs. // It allows a function caller differentiate unexpected program errors from errors caused by // invalid inputs. diff --git a/crypto/dkg_feldmanvss.go b/crypto/dkg_feldmanvss.go index 451c1b8a180..175fe5acb0a 100644 --- a/crypto/dkg_feldmanvss.go +++ b/crypto/dkg_feldmanvss.go @@ -3,7 +3,7 @@ package crypto -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #include "dkg_include.h" import "C" diff --git a/crypto/dkg_feldmanvssq.go b/crypto/dkg_feldmanvssq.go index 335ce6fc86d..ecb26f7d6e3 100644 --- a/crypto/dkg_feldmanvssq.go +++ b/crypto/dkg_feldmanvssq.go @@ -3,7 +3,7 @@ package crypto -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #include "dkg_include.h" import "C" diff --git a/crypto/dkg_jointfeldman.go b/crypto/dkg_jointfeldman.go index 7b63f88e810..d79379f7d83 100644 --- a/crypto/dkg_jointfeldman.go +++ b/crypto/dkg_jointfeldman.go @@ -3,7 +3,7 @@ package crypto -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #cgo LDFLAGS: -L${SRCDIR}/relic/build/lib -l relic_s // #include "dkg_include.h" import "C" diff --git a/crypto/ecdsa.go b/crypto/ecdsa.go index 1fc624c543d..dca3604570a 100644 --- a/crypto/ecdsa.go +++ b/crypto/ecdsa.go @@ -11,32 +11,28 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/sha256" "fmt" "math/big" "github.com/btcsuite/btcd/btcec/v2" + "golang.org/x/crypto/hkdf" "github.com/onflow/flow-go/crypto/hash" ) const ( - // ECDSA - - KeyGenSeedMaxLenECDSA = 2048 // large enough constant accepted by the implementation - // NIST P256 SignatureLenECDSAP256 = 64 PrKeyLenECDSAP256 = 32 // PubKeyLenECDSAP256 is the size of uncompressed points on P256 - PubKeyLenECDSAP256 = 64 - KeyGenSeedMinLenECDSAP256 = PrKeyLenECDSAP256 + (securityBits / 8) + PubKeyLenECDSAP256 = 64 // SECG secp256k1 SignatureLenECDSASecp256k1 = 64 PrKeyLenECDSASecp256k1 = 32 // PubKeyLenECDSASecp256k1 is the size of uncompressed points on secp256k1 - PubKeyLenECDSASecp256k1 = 64 - KeyGenSeedMinLenECDSASecp256k1 = PrKeyLenECDSASecp256k1 + (securityBits / 8) + PubKeyLenECDSASecp256k1 = 64 ) // ecdsaAlgo embeds SignAlgo @@ -181,7 +177,7 @@ func (a *ecdsaAlgo) signatureFormatCheck(sig Signature) bool { var one = new(big.Int).SetInt64(1) // goecdsaGenerateKey generates a public and private key pair -// for the crypto/ecdsa library using the input seed as input +// for the crypto/ecdsa library using the input seed func goecdsaGenerateKey(c elliptic.Curve, seed []byte) *ecdsa.PrivateKey { k := new(big.Int).SetBytes(seed) n := new(big.Int).Sub(c.Params().N, one) @@ -199,16 +195,34 @@ func goecdsaGenerateKey(c elliptic.Curve, seed []byte) *ecdsa.PrivateKey { // deterministically using the input seed. // // It is recommended to use a secure crypto RNG to generate the seed. -// The seed must have enough entropy and should be sampled uniformly at random. +// The seed must have enough entropy. func (a *ecdsaAlgo) generatePrivateKey(seed []byte) (PrivateKey, error) { - Nlen := bitsToBytes((a.curve.Params().N).BitLen()) - // use extra 128 bits to reduce the modular reduction bias - minSeedLen := Nlen + (securityBits / 8) - if len(seed) < minSeedLen || len(seed) > KeyGenSeedMaxLenECDSA { + if len(seed) < KeyGenSeedMinLen || len(seed) > KeyGenSeedMaxLen { return nil, invalidInputsErrorf("seed byte length should be between %d and %d", - minSeedLen, KeyGenSeedMaxLenECDSA) + KeyGenSeedMinLen, KeyGenSeedMaxLen) + } + + // use HKDF to extract the seed entropy and expand it into key bytes + + // use SHA2-256 as the building block H in HKDF + hashFunction := sha256.New + salt := []byte("") // HKDF salt + info := []byte("") // HKDF info + // use extra 128 bits to reduce the modular reduction bias + Nlen := bitsToBytes((a.curve.Params().N).BitLen()) + okmLength := Nlen + (securityBits / 8) + + // instantiate HKDF and extract okm + reader := hkdf.New(hashFunction, seed, salt, info) + okm := make([]byte, okmLength) + n, err := reader.Read(okm) + if err != nil || n != okmLength { + return nil, fmt.Errorf("key generation failed because of the HKDF reader, %d bytes were read: %w", + n, err) } - sk := goecdsaGenerateKey(a.curve, seed) + defer overwrite(okm) // overwrite okm + + sk := goecdsaGenerateKey(a.curve, okm) return &prKeyECDSA{ alg: a, goPrKey: sk, diff --git a/crypto/ecdsa_test.go b/crypto/ecdsa_test.go index 3c86a403847..c1ac2118f25 100644 --- a/crypto/ecdsa_test.go +++ b/crypto/ecdsa_test.go @@ -22,15 +22,6 @@ var ecdsaCurves = []SigningAlgorithm{ ECDSAP256, ECDSASecp256k1, } -var ecdsaMinSeed = map[SigningAlgorithm]int{ - ECDSAP256: KeyGenSeedMinLenECDSAP256, - ECDSASecp256k1: KeyGenSeedMinLenECDSASecp256k1, -} - -var ecdsaMaxSeed = map[SigningAlgorithm]int{ - ECDSAP256: KeyGenSeedMaxLenECDSA, - ECDSASecp256k1: KeyGenSeedMaxLenECDSA, -} var ecdsaPrKeyLen = map[SigningAlgorithm]int{ ECDSAP256: PrKeyLenECDSAP256, ECDSASecp256k1: PrKeyLenECDSASecp256k1, @@ -50,7 +41,7 @@ func TestECDSA(t *testing.T) { for _, curve := range ecdsaCurves { t.Logf("Testing ECDSA for curve %s", curve) // test key generation seed limits - testKeyGenSeed(t, curve, ecdsaMinSeed[curve], ecdsaMaxSeed[curve]) + testKeyGenSeed(t, curve, KeyGenSeedMinLen, KeyGenSeedMaxLen) // test consistency halg := hash.NewSHA3_256() testGenSignVerify(t, curve, halg) @@ -72,9 +63,9 @@ func TestECDSAHasher(t *testing.T) { for _, curve := range ecdsaCurves { // generate a key pair - seed := make([]byte, ecdsaMinSeed[curve]) + seed := make([]byte, KeyGenSeedMinLen) n, err := rand.Read(seed) - require.Equal(t, n, ecdsaMinSeed[curve]) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk, err := GeneratePrivateKey(curve, seed) require.NoError(t, err) @@ -155,9 +146,9 @@ func TestECDSAUtils(t *testing.T) { for _, curve := range ecdsaCurves { // generate a key pair - seed := make([]byte, ecdsaMinSeed[curve]) + seed := make([]byte, KeyGenSeedMinLen) n, err := rand.Read(seed) - require.Equal(t, n, ecdsaMinSeed[curve]) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk, err := GeneratePrivateKey(curve, seed) require.NoError(t, err) diff --git a/crypto/go.mod b/crypto/go.mod index c7fe54f9ff5..57c20ef9341 100644 --- a/crypto/go.mod +++ b/crypto/go.mod @@ -6,10 +6,8 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.8.0 - github.com/supranational/blst v0.3.10 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d gonum.org/v1/gonum v0.6.1 - pgregory.net/rapid v0.4.7 ) require ( diff --git a/crypto/go.sum b/crypto/go.sum index 19a05d05d6d..181f9b302c0 100644 --- a/crypto/go.sum +++ b/crypto/go.sum @@ -28,8 +28,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= -github.com/supranational/blst v0.3.10/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -54,6 +52,4 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= -pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/crypto/hash/hash_test.go b/crypto/hash/hash_test.go index f2eef310e94..fa61bbf470e 100644 --- a/crypto/hash/hash_test.go +++ b/crypto/hash/hash_test.go @@ -1,6 +1,8 @@ package hash import ( + "crypto/sha256" + "crypto/sha512" "encoding/hex" "math/rand" "testing" @@ -160,7 +162,46 @@ func TestHashersAPI(t *testing.T) { } } -// TestSHA3 is a specific test of SHA3-256 and SHA3-388. +// TestSHA2 is a specific test of SHA2-256 and SHA2-384. +// It compares the hashes of random data of different lengths to +// the output of standard Go sha2. +func TestSHA2(t *testing.T) { + r := time.Now().UnixNano() + rand.Seed(r) + t.Logf("math rand seed is %d", r) + + t.Run("SHA2_256", func(t *testing.T) { + for i := 0; i < 5000; i++ { + value := make([]byte, i) + rand.Read(value) + expected := sha256.Sum256(value) + + // test hash computation using the hasher + hasher := NewSHA2_256() + h := hasher.ComputeHash(value) + assert.Equal(t, expected[:], []byte(h)) + + // test hash computation using the light api + var res [HashLenSHA2_256]byte + ComputeSHA2_256(&res, value) + assert.Equal(t, expected[:], res[:]) + } + }) + + t.Run("SHA2_384", func(t *testing.T) { + for i := 0; i < 5000; i++ { + value := make([]byte, i) + rand.Read(value) + expected := sha512.Sum384(value) + + hasher := NewSHA2_384() + h := hasher.ComputeHash(value) + assert.Equal(t, expected[:], []byte(h)) + } + }) +} + +// TestSHA3 is a specific test of SHA3-256 and SHA3-384. // It compares the hashes of random data of different lengths to // the output of standard Go sha3. func TestSHA3(t *testing.T) { @@ -224,7 +265,7 @@ func TestKeccak(t *testing.T) { // Benchmark of all hashers' ComputeHash function func BenchmarkComputeHash(b *testing.B) { - m := make([]byte, 64) + m := make([]byte, 32) rand.Read(m) b.Run("SHA2_256", func(b *testing.B) { @@ -236,6 +277,15 @@ func BenchmarkComputeHash(b *testing.B) { b.StopTimer() }) + b.Run("SHA2_256_light", func(b *testing.B) { + var h [HashLenSHA2_256]byte + b.ResetTimer() + for i := 0; i < b.N; i++ { + ComputeSHA2_256(&h, m) + } + b.StopTimer() + }) + b.Run("SHA2_384", func(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/crypto/hash/sha2.go b/crypto/hash/sha2.go index 15e2d618783..3362face47a 100644 --- a/crypto/hash/sha2.go +++ b/crypto/hash/sha2.go @@ -6,7 +6,7 @@ import ( "hash" ) -// sha2_256Algo, embeds commonHasher +// sha2_256Algo type sha2_256Algo struct { hash.Hash } @@ -37,7 +37,7 @@ func (s *sha2_256Algo) SumHash() Hash { return s.Sum(nil) } -// sha2_384Algo, embeds commonHasher +// sha2_384Algo type sha2_384Algo struct { hash.Hash } @@ -66,3 +66,13 @@ func (s *sha2_384Algo) ComputeHash(data []byte) Hash { func (s *sha2_384Algo) SumHash() Hash { return s.Sum(nil) } + +// ComputeSHA2_256 computes the SHA2-256 (commonly known as SHA256) +// digest of data and copies the result to the result buffer. +// +// The function is not part of the Hasher API. It is a pure function +// for simple computation of a hash with minimal heap allocations. +func ComputeSHA2_256(result *[HashLenSHA2_256]byte, data []byte) { + hash := sha256.Sum256(data) + copy(result[:], hash[:]) +} diff --git a/crypto/hash/sha3.go b/crypto/hash/sha3.go index 9ebc07922dd..f5b6cd9fce4 100644 --- a/crypto/hash/sha3.go +++ b/crypto/hash/sha3.go @@ -34,9 +34,8 @@ func NewSHA3_384() Hasher { // ComputeSHA3_256 computes the SHA3-256 digest of data // and copies the result to the result buffer. // -// The function is not part of the Hasher API. It is a light API -// that allows a simple computation of a hash and minimizes -// heap allocations. +// The function is not part of the Hasher API. It is a pure function +// for simple computation of a hash with minimal heap allocations. func ComputeSHA3_256(result *[HashLenSHA3_256]byte, data []byte) { state := &spongeState{ rate: rateSHA3_256, diff --git a/crypto/sign.go b/crypto/sign.go index 06ebbae85ff..68196acba2d 100644 --- a/crypto/sign.go +++ b/crypto/sign.go @@ -104,15 +104,14 @@ func SignatureFormatCheck(algo SigningAlgorithm, s Signature) (bool, error) { // GeneratePrivateKey generates a private key of the algorithm using the entropy of the given seed. // +// The seed minimum length is 32 bytes and it should have enough entropy. // It is recommended to use a secure crypto RNG to generate the seed. -// The seed must have enough entropy and should be sampled uniformly at random. // // The function returns: // - (false, invalidInputsErrors) if the signing algorithm is not supported or -// -// if the seed length is not valid (too short or too long) +// if the seed length is not valid (less than 32 bytes or larger than 256 bytes) // - (false, error) if an unexpected error occurs -// - (sk, nil) otherwise +// - (sk, nil) if key generation was successful func GeneratePrivateKey(algo SigningAlgorithm, seed []byte) (PrivateKey, error) { signer, err := newSigner(algo) if err != nil { diff --git a/crypto/sign_test_utils.go b/crypto/sign_test_utils.go index 0f5d38a1d97..e9198a0c7b5 100644 --- a/crypto/sign_test_utils.go +++ b/crypto/sign_test_utils.go @@ -105,23 +105,52 @@ func testGenSignVerify(t *testing.T, salg SigningAlgorithm, halg hash.Hasher) { } } +// tests the key generation constraints with regards to the input seed, mainly +// the seed length constraints and the result determinicity. func testKeyGenSeed(t *testing.T, salg SigningAlgorithm, minLen int, maxLen int) { - // valid seed lengths - seed := make([]byte, minLen) - _, err := GeneratePrivateKey(salg, seed) - assert.NoError(t, err) - seed = make([]byte, maxLen) - _, err = GeneratePrivateKey(salg, seed) - assert.NoError(t, err) - // invalid seed lengths - seed = make([]byte, minLen-1) - _, err = GeneratePrivateKey(salg, seed) - assert.Error(t, err) - assert.True(t, IsInvalidInputsError(err)) - seed = make([]byte, maxLen+1) - _, err = GeneratePrivateKey(salg, seed) - assert.Error(t, err) - assert.True(t, IsInvalidInputsError(err)) + t.Run("seed length check", func(t *testing.T) { + // valid seed lengths + seed := make([]byte, minLen) + _, err := GeneratePrivateKey(salg, seed) + assert.NoError(t, err) + if maxLen > 0 { + seed = make([]byte, maxLen) + _, err = GeneratePrivateKey(salg, seed) + assert.NoError(t, err) + } + // invalid seed lengths + seed = make([]byte, minLen-1) + _, err = GeneratePrivateKey(salg, seed) + assert.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + if maxLen > 0 { + seed = make([]byte, maxLen+1) + _, err = GeneratePrivateKey(salg, seed) + assert.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + } + }) + + t.Run("deterministic generation", func(t *testing.T) { + r := time.Now().UnixNano() + mrand.Seed(r) + t.Logf("math rand seed is %d", r) + // same seed results in the same key + seed := make([]byte, minLen) + read, err := mrand.Read(seed) + require.Equal(t, read, minLen) + require.NoError(t, err) + sk1, err := GeneratePrivateKey(salg, seed) + require.NoError(t, err) + sk2, err := GeneratePrivateKey(salg, seed) + require.NoError(t, err) + assert.True(t, sk1.Equals(sk2)) + // different seed results in a different key + seed[0] ^= 1 // alter a seed bit + sk2, err = GeneratePrivateKey(salg, seed) + require.NoError(t, err) + assert.False(t, sk1.Equals(sk2)) + }) } func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { diff --git a/crypto/spock.go b/crypto/spock.go index 2487f39ce1b..a4087316319 100644 --- a/crypto/spock.go +++ b/crypto/spock.go @@ -6,7 +6,7 @@ package crypto // SPoCK design based on the BLS signature scheme. // BLS is using BLS12-381 curve and the same settings in bls.go. -// #cgo CFLAGS: -g -Wall -std=c99 +// #cgo CFLAGS: // #cgo LDFLAGS: -L${SRCDIR}/relic/build/lib -l relic_s // #include "bls_include.h" import "C" diff --git a/crypto/spock_test.go b/crypto/spock_test.go index e617c6d0518..45db590f04e 100644 --- a/crypto/spock_test.go +++ b/crypto/spock_test.go @@ -13,11 +13,11 @@ import ( func TestSPOCKProveVerifyAgainstData(t *testing.T) { // test the consistency with different data - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed := make([]byte, KeyGenSeedMinLen) data := make([]byte, 100) n, err := rand.Read(seed) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) @@ -82,8 +82,8 @@ func TestSPOCKProveVerifyAgainstData(t *testing.T) { // tests of happy and unhappy paths of SPOCKVerify func TestSPOCKProveVerify(t *testing.T) { // test the consistency with different data - seed1 := make([]byte, KeyGenSeedMinLenBLSBLS12381) - seed2 := make([]byte, KeyGenSeedMinLenBLSBLS12381) + seed1 := make([]byte, KeyGenSeedMinLen) + seed2 := make([]byte, KeyGenSeedMinLen) data := make([]byte, 100) // data @@ -91,13 +91,13 @@ func TestSPOCKProveVerify(t *testing.T) { require.NoError(t, err) // sk1 n, err := rand.Read(seed1) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk1, err := GeneratePrivateKey(BLSBLS12381, seed1) require.NoError(t, err) // sk2 n, err = rand.Read(seed2) - require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) + require.Equal(t, n, KeyGenSeedMinLen) require.NoError(t, err) sk2, err := GeneratePrivateKey(BLSBLS12381, seed2) require.NoError(t, err) diff --git a/docs/ComponentInteraction.png b/docs/ComponentInteraction.png new file mode 100644 index 00000000000..f483309cbd4 Binary files /dev/null and b/docs/ComponentInteraction.png differ diff --git a/docs/HotStuffEventLoop.png b/docs/HotStuffEventLoop.png deleted file mode 100644 index 578152ad05a..00000000000 Binary files a/docs/HotStuffEventLoop.png and /dev/null differ diff --git a/docs/PaceMaker.png b/docs/PaceMaker.png index e721ee642dc..de852e86a39 100644 Binary files a/docs/PaceMaker.png and b/docs/PaceMaker.png differ diff --git a/docs/StateMachine.png b/docs/StateMachine.png index 167de17e150..68daa8ed432 100644 Binary files a/docs/StateMachine.png and b/docs/StateMachine.png differ diff --git a/docs/StateMachine_wirth_notifications.png b/docs/StateMachine_wirth_notifications.png deleted file mode 100644 index b9e55155293..00000000000 Binary files a/docs/StateMachine_wirth_notifications.png and /dev/null differ diff --git a/docs/StateMachine_with_notifications.png b/docs/StateMachine_with_notifications.png new file mode 100644 index 00000000000..1cf85ac220d Binary files /dev/null and b/docs/StateMachine_with_notifications.png differ diff --git a/engine/access/access_test.go b/engine/access/access_test.go index 1c930a723d7..d77dcdc1b76 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -50,6 +50,7 @@ type Suite struct { state *protocol.State snapshot *protocol.Snapshot epochQuery *protocol.EpochQuery + params *protocol.Params signerIndicesDecoder *hsmock.BlockSignerDecoder signerIds flow.IdentifierList log zerolog.Logger @@ -58,6 +59,7 @@ type Suite struct { collClient *accessmock.AccessAPIClient execClient *accessmock.ExecutionAPIClient me *module.Local + rootBlock *flow.Header chainID flow.ChainID metrics *metrics.NoopCollector backend *backend.Backend @@ -80,10 +82,11 @@ func (suite *Suite) SetupTest() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() - header := unittest.BlockHeaderFixture() - params := new(protocol.Params) - params.On("Root").Return(header, nil) - suite.state.On("Params").Return(params).Maybe() + suite.rootBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) + suite.params = new(protocol.Params) + suite.params.On("Root").Return(suite.rootBlock, nil) + suite.params.On("SporkRootBlockHeight").Return(suite.rootBlock.Height, nil) + suite.state.On("Params").Return(suite.params).Maybe() suite.collClient = new(accessmock.AccessAPIClient) suite.execClient = new(accessmock.ExecutionAPIClient) @@ -864,9 +867,10 @@ func (suite *Suite) createChain() (flow.Block, flow.Collection) { epoch.On("ClusterByChainID", mock.Anything).Return(cluster, nil) epochs := new(protocol.EpochQuery) epochs.On("Current").Return(epoch) - snap := new(protocol.Snapshot) - snap.On("Epochs").Return(epochs) - snap.On("SealingSegment").Return(&flow.SealingSegment{Blocks: unittest.BlockFixtures(2)}, nil).Maybe() + snap := protocol.NewSnapshot(suite.T()) + snap.On("Epochs").Return(epochs).Maybe() + snap.On("Params").Return(suite.params).Maybe() + snap.On("Head").Return(block.Header, nil).Maybe() suite.state.On("AtBlockID", mock.Anything).Return(snap).Once() // initial height lookup in ingestion engine suite.state.On("AtBlockID", refBlockID).Return(snap) diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index 40381c93c11..b4588397660 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -7,6 +7,7 @@ import ( "time" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -62,8 +63,8 @@ func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { if identity.NetworkPubKey == nil { connection, err = grpc.Dial( identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithInsecure(), //nolint:staticcheck + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials()), backend.WithClientUnaryInterceptor(timeout)) if err != nil { return err @@ -76,7 +77,7 @@ func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { connection, err = grpc.Dial( identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), backend.WithClientUnaryInterceptor(timeout)) if err != nil { @@ -291,10 +292,11 @@ type FlowAccessAPIForwarder struct { upstream []access.AccessAPIClient connections []*grpc.ClientConn timeout time.Duration + maxMsgSize uint } -func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration) (*FlowAccessAPIForwarder, error) { - forwarder := &FlowAccessAPIForwarder{} +func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*FlowAccessAPIForwarder, error) { + forwarder := &FlowAccessAPIForwarder{maxMsgSize: maxMsgSize} err := forwarder.setFlowAccessAPI(identities, timeout) return forwarder, err } diff --git a/engine/access/ingestion/engine.go b/engine/access/ingestion/engine.go index c1b1d2865f8..58b0617a2bd 100644 --- a/engine/access/ingestion/engine.go +++ b/engine/access/ingestion/engine.go @@ -75,6 +75,7 @@ type Engine struct { collections storage.Collections transactions storage.Transactions executionReceipts storage.ExecutionReceipts + maxReceiptHeight uint64 executionResults storage.ExecutionResults // metrics @@ -150,6 +151,7 @@ func New( transactions: transactions, executionResults: executionResults, executionReceipts: executionReceipts, + maxReceiptHeight: 0, transactionMetrics: transactionMetrics, collectionsToMarkFinalized: collectionsToMarkFinalized, collectionsToMarkExecuted: collectionsToMarkExecuted, @@ -184,40 +186,31 @@ func New( } func (e *Engine) Start(parent irrecoverable.SignalerContext) { - rootBlock, err := e.state.Params().Root() + err := e.initLastFullBlockHeightIndex() if err != nil { - parent.Throw(fmt.Errorf("failed to get root block: %w", err)) + parent.Throw(fmt.Errorf("unexpected error initializing full block index: %w", err)) } - // if spork root snapshot - rootSnapshot := e.state.AtBlockID(rootBlock.ID()) + e.ComponentManager.Start(parent) +} - isSporkRootSnapshot, err := protocol.IsSporkRootSnapshot(rootSnapshot) +// initializeLastFullBlockHeightIndex initializes the index of full blocks +// (blocks for which we have ingested all collections) to the root block height. +// This means that the Access Node will ingest all collections for all blocks +// ingested after state bootstrapping is complete (all blocks received from the network). +// If the index has already been initialized, this is a no-op. +// No errors are expected during normal operation. +func (e *Engine) initLastFullBlockHeightIndex() error { + rootBlock, err := e.state.Params().Root() if err != nil { - parent.Throw(fmt.Errorf("could not check if root snapshot is a spork root snapshot: %w", err)) + return fmt.Errorf("failed to get root block: %w", err) } - - // This is useful for dynamically bootstrapped access node, they will request missing collections. In order to ensure all txs - // from the missing collections can be verified, we must ensure they are referencing to known blocks. - // That's why we set the full block height to be rootHeight + TransactionExpiry, so that we only request missing collections - // in blocks above that height. - if isSporkRootSnapshot { - // for snapshot with a single block in the sealing segment the first full block is the root block. - err := e.blocks.InsertLastFullBlockHeightIfNotExists(rootBlock.Height) - if err != nil { - parent.Throw(fmt.Errorf("failed to update last full block height during ingestion engine startup: %w", err)) - } - } else { - // for midspork snapshots with a sealing segment that has more than 1 block add the transaction expiry to the root block height to avoid - // requesting resources for blocks below the expiry. - firstFullHeight := rootBlock.Height + flow.DefaultTransactionExpiry - err := e.blocks.InsertLastFullBlockHeightIfNotExists(firstFullHeight) - if err != nil { - parent.Throw(fmt.Errorf("failed to update last full block height during ingestion engine startup: %w", err)) - } + err = e.blocks.InsertLastFullBlockHeightIfNotExists(rootBlock.Height) + if err != nil { + return fmt.Errorf("failed to update last full block height during ingestion engine startup: %w", err) } - e.ComponentManager.Start(parent) + return nil } func (e *Engine) processBackground(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { diff --git a/engine/access/rest/middleware/metrics.go b/engine/access/rest/middleware/metrics.go new file mode 100644 index 00000000000..c0d51d36eb6 --- /dev/null +++ b/engine/access/rest/middleware/metrics.go @@ -0,0 +1,32 @@ +package middleware + +import ( + "net/http" + + "github.com/onflow/flow-go/module/metrics" + + "github.com/slok/go-http-metrics/middleware" + "github.com/slok/go-http-metrics/middleware/std" + + metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" + + "github.com/gorilla/mux" +) + +func MetricsMiddleware() mux.MiddlewareFunc { + r := metrics.NewRestCollector(metricsProm.Config{Prefix: "access_rest_api"}) + metricsMiddleware := middleware.New(middleware.Config{Recorder: r}) + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + // This is a custom metric being called on every http request + r.AddTotalRequests(req.Context(), req.Method, req.URL.Path) + + // Modify the writer + respWriter := &responseWriter{w, http.StatusOK} + + // Record go-http-metrics/middleware metrics and continue to the next handler + std.Handler("", metricsMiddleware, next).ServeHTTP(respWriter, req) + }) + } +} diff --git a/engine/access/rest/request/address.go b/engine/access/rest/request/address.go index 173ed69885c..4dc2eebbbd7 100644 --- a/engine/access/rest/request/address.go +++ b/engine/access/rest/request/address.go @@ -8,20 +8,13 @@ import ( "github.com/onflow/flow-go/model/flow" ) -type Address flow.Address - -func (a *Address) Parse(raw string) error { +func ParseAddress(raw string) (flow.Address, error) { raw = strings.ReplaceAll(raw, "0x", "") // remove 0x prefix valid, _ := regexp.MatchString(`^[0-9a-fA-F]{16}$`, raw) if !valid { - return fmt.Errorf("invalid address") + return flow.EmptyAddress, fmt.Errorf("invalid address") } - *a = Address(flow.HexToAddress(raw)) - return nil -} - -func (a Address) Flow() flow.Address { - return flow.Address(a) + return flow.HexToAddress(raw), nil } diff --git a/engine/access/rest/request/address_test.go b/engine/access/rest/request/address_test.go index 743347adc10..3f0f7a45bf2 100644 --- a/engine/access/rest/request/address_test.go +++ b/engine/access/rest/request/address_test.go @@ -8,7 +8,6 @@ import ( ) func TestAddress_InvalidParse(t *testing.T) { - var address Address inputs := []string{ "0x1", "", @@ -19,13 +18,12 @@ func TestAddress_InvalidParse(t *testing.T) { } for _, input := range inputs { - err := address.Parse(input) + _, err := ParseAddress(input) assert.EqualError(t, err, "invalid address") } } func TestAddress_ValidParse(t *testing.T) { - var address Address inputs := []string{ "f8d6e0586b0a20c7", "f3ad66eea58c97d2", @@ -33,8 +31,8 @@ func TestAddress_ValidParse(t *testing.T) { } for _, input := range inputs { - err := address.Parse(input) + address, err := ParseAddress(input) assert.NoError(t, err) - assert.Equal(t, strings.ReplaceAll(input, "0x", ""), address.Flow().String()) + assert.Equal(t, strings.ReplaceAll(input, "0x", ""), address.String()) } } diff --git a/engine/access/rest/request/get_account.go b/engine/access/rest/request/get_account.go index 2bc5d4a95c9..fde9cf6d6e7 100644 --- a/engine/access/rest/request/get_account.go +++ b/engine/access/rest/request/get_account.go @@ -20,8 +20,7 @@ func (g *GetAccount) Build(r *Request) error { } func (g *GetAccount) Parse(rawAddress string, rawHeight string) error { - var address Address - err := address.Parse(rawAddress) + address, err := ParseAddress(rawAddress) if err != nil { return err } @@ -32,7 +31,7 @@ func (g *GetAccount) Parse(rawAddress string, rawHeight string) error { return err } - g.Address = address.Flow() + g.Address = address g.Height = height.Flow() // default to last block diff --git a/engine/access/rest/request/proposal_key.go b/engine/access/rest/request/proposal_key.go index 7add979dc21..1b5d5c13dc6 100644 --- a/engine/access/rest/request/proposal_key.go +++ b/engine/access/rest/request/proposal_key.go @@ -11,8 +11,7 @@ import ( type ProposalKey flow.ProposalKey func (p *ProposalKey) Parse(raw models.ProposalKey) error { - var address Address - err := address.Parse(raw.Address) + address, err := ParseAddress(raw.Address) if err != nil { return err } @@ -28,7 +27,7 @@ func (p *ProposalKey) Parse(raw models.ProposalKey) error { } *p = ProposalKey(flow.ProposalKey{ - Address: address.Flow(), + Address: address, KeyIndex: keyIndex, SequenceNumber: seqNumber, }) diff --git a/engine/access/rest/request/signatures.go b/engine/access/rest/request/signatures.go index e4f77b4c7c5..8afa83bf072 100644 --- a/engine/access/rest/request/signatures.go +++ b/engine/access/rest/request/signatures.go @@ -15,8 +15,7 @@ func (s *TransactionSignature) Parse( rawKeyIndex string, rawSignature string, ) error { - var address Address - err := address.Parse(rawAddress) + address, err := ParseAddress(rawAddress) if err != nil { return err } @@ -33,7 +32,7 @@ func (s *TransactionSignature) Parse( } *s = TransactionSignature(flow.TransactionSignature{ - Address: address.Flow(), + Address: address, KeyIndex: keyIndex, Signature: signature, }) diff --git a/engine/access/rest/request/transaction.go b/engine/access/rest/request/transaction.go index 52d22f7f66c..10564a34ea1 100644 --- a/engine/access/rest/request/transaction.go +++ b/engine/access/rest/request/transaction.go @@ -50,21 +50,19 @@ func (t *Transaction) Parse(raw io.Reader, chain flow.Chain) error { return err } - var payer Address - err = payer.Parse(tx.Payer) + payer, err := ParseAddress(tx.Payer) if err != nil { return fmt.Errorf("invalid payer: %w", err) } auths := make([]flow.Address, len(tx.Authorizers)) for i, auth := range tx.Authorizers { - var a Address - err := a.Parse(auth) + a, err := ParseAddress(auth) if err != nil { return err } - auths[i] = a.Flow() + auths[i] = a } var proposal ProposalKey @@ -108,7 +106,7 @@ func (t *Transaction) Parse(raw io.Reader, chain flow.Chain) error { Arguments: args.Flow(), GasLimit: gasLimit, ProposalKey: proposal.Flow(), - Payer: payer.Flow(), + Payer: payer, Authorizers: auths, PayloadSignatures: payloadSigs.Flow(), EnvelopeSignatures: envelopeSigs.Flow(), diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index 3b8168b2283..d750c000578 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -20,6 +20,7 @@ func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain) (*mu v1SubRouter.Use(middleware.LoggingMiddleware(logger)) v1SubRouter.Use(middleware.QueryExpandable()) v1SubRouter.Use(middleware.QuerySelect()) + v1SubRouter.Use(middleware.MetricsMiddleware()) linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index 59e914210f5..0daae2034ed 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -138,19 +138,8 @@ func (b *backendAccounts) getAccountFromAnyExeNode(ctx context.Context, execNode Msg("failed to execute GetAccount") errors = multierror.Append(errors, err) } - // if we made it till here means there was at least one error - errToReturn := errors.ErrorOrNil() - - // if there were an any errors other than codes.NotFound, return those - for _, err := range errors.Errors { - errStatus, _ := status.FromError(err) - if errStatus.Code() != codes.NotFound { - return nil, status.Errorf(codes.Internal, "failed to get account from the execution node: %v", errToReturn) - } - } - // if all errors were codes.NotFound, then return a codes.NotFound error wrapping all those errors - return nil, status.Errorf(codes.NotFound, "failed to get account from the execution node: %v", errToReturn) + return nil, rpc.ConvertMultiError(errors, "failed to get account from the execution node", codes.Internal) } func (b *backendAccounts) tryGetAccount(ctx context.Context, execNode *flow.Identity, req *execproto.GetAccountAtBlockIDRequest) (*execproto.GetAccountAtBlockIDResponse, error) { diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index bc2098a24e3..8af4b915ef0 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -139,7 +139,7 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( resp, successfulNode, err = b.getEventsFromAnyExeNode(ctx, execNodes, req) if err != nil { b.log.Error().Err(err).Msg("failed to retrieve events from execution nodes") - return nil, status.Errorf(codes.Internal, "failed to retrieve events from execution nodes %s: %v", execNodes, err) + return nil, err } b.log.Trace(). Str("execution_id", successfulNode.String()). @@ -203,7 +203,7 @@ func (b *backendEvents) getEventsFromAnyExeNode(ctx context.Context, } errors = multierror.Append(errors, err) } - return nil, nil, errors.ErrorOrNil() + return nil, nil, rpc.ConvertMultiError(errors, "failed to retrieve events from execution nodes", codes.Internal) } func (b *backendEvents) tryGetEvents(ctx context.Context, diff --git a/engine/access/rpc/backend/backend_network.go b/engine/access/rpc/backend/backend_network.go index 78205f5d986..c3bdaf92c93 100644 --- a/engine/access/rpc/backend/backend_network.go +++ b/engine/access/rpc/backend/backend_network.go @@ -73,7 +73,7 @@ func (b *backendNetwork) getValidSnapshot(snapshot protocol.Snapshot, blocksVisi return nil, fmt.Errorf("failed to get counter and phase at highest block in the segment: %w", err) } - counterAtLowest, phaseAtLowest, err := b.getCounterAndPhase(segment.Lowest().Header.Height) + counterAtLowest, phaseAtLowest, err := b.getCounterAndPhase(segment.Sealed().Header.Height) if err != nil { return nil, fmt.Errorf("failed to get counter and phase at lowest block in the segment: %w", err) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index 79f6c9582fd..a8613dcd68b 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -146,11 +146,13 @@ func (b *backendScripts) executeScriptOnExecutionNode( } errors = multierror.Append(errors, err) } + errToReturn := errors.ErrorOrNil() if errToReturn != nil { b.log.Error().Err(err).Msg("script execution failed for execution node internal reasons") } - return nil, errToReturn + + return nil, rpc.ConvertMultiError(errors, "failed to execute script on execution nodes", codes.Internal) } // shouldLogScript checks if the script hash is unique in the time window diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 796c3cba5c2..4494fc6c22f 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -223,7 +223,7 @@ func (b *backendTransactions) GetTransactionsByBlockID( systemTx, err := blueprints.SystemChunkTransaction(b.chainID.Chain()) if err != nil { - return nil, fmt.Errorf("could not get system chunk transaction: %w", err) + return nil, status.Errorf(codes.Internal, "could not get system chunk transaction: %v", err) } transactions = append(transactions, systemTx) @@ -323,10 +323,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( resp, err := b.getTransactionResultsByBlockIDFromAnyExeNode(ctx, execNodes, req) if err != nil { - if status.Code(err) == codes.NotFound { - return nil, err - } - return nil, status.Errorf(codes.Internal, "failed to retrieve result from execution node: %v", err) + return nil, err } results := make([]*access.TransactionResult, 0, len(resp.TransactionResults)) @@ -385,7 +382,7 @@ func (b *backendTransactions) GetTransactionResultsByBlockID( systemTx, err := blueprints.SystemChunkTransaction(b.chainID.Chain()) if err != nil { - return nil, fmt.Errorf("could not get system chunk transaction: %w", err) + return nil, status.Errorf(codes.Internal, "could not get system chunk transaction: %v", err) } systemTxResult := resp.TransactionResults[len(resp.TransactionResults)-1] systemTxStatus, err := b.deriveTransactionStatus(systemTx, true, block) @@ -436,10 +433,7 @@ func (b *backendTransactions) GetTransactionResultByIndex( resp, err := b.getTransactionResultByIndexFromAnyExeNode(ctx, execNodes, req) if err != nil { - if status.Code(err) == codes.NotFound { - return nil, err - } - return nil, status.Errorf(codes.Internal, "failed to retrieve result from execution node: %v", err) + return nil, err } // tx body is irrelevant to status if it's in an executed block @@ -585,8 +579,12 @@ func (b *backendTransactions) getHistoricalTransaction( txResp, err := historicalNode.GetTransaction(ctx, &accessproto.GetTransactionRequest{Id: txID[:]}) if err == nil { tx, err := convert.MessageToTransaction(txResp.Transaction, b.chainID.Chain()) + if err != nil { + return nil, status.Errorf(codes.Internal, "could not convert transaction: %v", err) + } + // Found on a historical node. Report - return &tx, err + return &tx, nil } // Otherwise, if not found, just continue if status.Code(err) == codes.NotFound { @@ -605,15 +603,18 @@ func (b *backendTransactions) getHistoricalTransactionResult( result, err := historicalNode.GetTransactionResult(ctx, &accessproto.GetTransactionRequest{Id: txID[:]}) if err == nil { // Found on a historical node. Report + if result.GetStatus() == entities.TransactionStatus_UNKNOWN { + // We've moved to returning Status UNKNOWN instead of an error with the NotFound status, + // Therefore we should continue and look at the next access node for answers. + continue + } + if result.GetStatus() == entities.TransactionStatus_PENDING { // This is on a historical node. No transactions from it will ever be // executed, therefore we should consider this expired result.Status = entities.TransactionStatus_EXPIRED - } else if result.GetStatus() == entities.TransactionStatus_UNKNOWN { - // We've moved to returning Status UNKNOWN instead of an error with the NotFound status, - // Therefore we should continue and look at the next access node for answers. - continue } + return access.MessageToTransactionResult(result), nil } // Otherwise, if not found, just continue @@ -657,10 +658,7 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( resp, err := b.getTransactionResultFromAnyExeNode(ctx, execNodes, req) if err != nil { - if status.Code(err) == codes.NotFound { - return nil, 0, "", err - } - return nil, 0, "", status.Errorf(codes.Internal, "failed to retrieve result from execution node: %v", err) + return nil, 0, "", err } events := convert.MessagesToEvents(resp.GetEvents()) @@ -701,7 +699,7 @@ func (b *backendTransactions) getTransactionResultFromAnyExeNode( } errs = multierror.Append(errs, err) } - return nil, errs.ErrorOrNil() + return nil, rpc.ConvertMultiError(errs, "failed to retrieve result from execution node", codes.Internal) } func (b *backendTransactions) tryGetTransactionResult( @@ -733,6 +731,7 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( var errs *multierror.Error defer func() { + // log the errors if err := errs.ErrorOrNil(); err != nil { b.log.Err(errs).Msg("failed to get transaction results from execution nodes") } @@ -758,8 +757,7 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromAnyExeNode( errs = multierror.Append(errs, err) } - // log the errors - return nil, errs.ErrorOrNil() + return nil, rpc.ConvertMultiError(errs, "failed to retrieve result from execution node", codes.Internal) } func (b *backendTransactions) tryGetTransactionResultsByBlockID( @@ -818,7 +816,7 @@ func (b *backendTransactions) getTransactionResultByIndexFromAnyExeNode( errs = multierror.Append(errs, err) } - return nil, errs.ErrorOrNil() + return nil, rpc.ConvertMultiError(errs, "failed to retrieve result from execution node", codes.Internal) } func (b *backendTransactions) tryGetTransactionResultByIndex( diff --git a/engine/access/rpc/backend/connection_factory.go b/engine/access/rpc/backend/connection_factory.go index 40e89c8a7a6..63ead3d3e32 100644 --- a/engine/access/rpc/backend/connection_factory.go +++ b/engine/access/rpc/backend/connection_factory.go @@ -18,7 +18,6 @@ import ( "google.golang.org/grpc/keepalive" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/utils/grpcutils" ) // DefaultClientTimeout is used when making a GRPC request to a collection node or an execution node @@ -58,6 +57,7 @@ type ConnectionFactoryImpl struct { ExecutionNodeGRPCTimeout time.Duration ConnectionsCache *lru.Cache CacheSize uint + MaxMsgSize uint AccessMetrics module.AccessMetrics Log zerolog.Logger mutex sync.Mutex @@ -90,7 +90,7 @@ func (cf *ConnectionFactoryImpl) createConnection(address string, timeout time.D // https://grpc.io/blog/grpc-on-http2/#keeping-connections-alive conn, err := grpc.Dial( address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(cf.MaxMsgSize))), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithKeepaliveParams(keepaliveParams), WithClientUnaryInterceptor(timeout)) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 3c882a75b1b..4f76f28863c 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/grpcutils" ) // Config defines the configurable options for the access node server @@ -39,8 +38,8 @@ type Config struct { RESTListenAddr string // the REST server address as ip:port (if empty the REST server will not be started) CollectionAddr string // the address of the upstream collection node HistoricalAccessAddrs string // the list of all access nodes from previous spork - MaxMsgSize int // GRPC max message size - MaxExecutionDataMsgSize int // GRPC max message size for block execution data + MaxMsgSize uint // GRPC max message size + MaxExecutionDataMsgSize uint // GRPC max message size for block execution data ExecutionClientTimeout time.Duration // execution API GRPC client timeout CollectionClientTimeout time.Duration // collection API GRPC client timeout ConnectionPoolSize uint // size of the cache for storing collection and execution connections @@ -94,14 +93,10 @@ func NewBuilder(log zerolog.Logger, log = log.With().Str("engine", "rpc").Logger() - if config.MaxMsgSize == 0 { - config.MaxMsgSize = grpcutils.DefaultMaxMsgSize - } - // create a GRPC server to serve GRPC clients grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(config.MaxMsgSize), - grpc.MaxSendMsgSize(config.MaxMsgSize), + grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors @@ -165,6 +160,7 @@ func NewBuilder(log zerolog.Logger, ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, ConnectionsCache: cache, CacheSize: cacheSize, + MaxMsgSize: config.MaxMsgSize, AccessMetrics: accessMetrics, Log: log, } diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 4f8b5fe6a09..59f292cf80c 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" accessmock "github.com/onflow/flow-go/engine/access/mock" @@ -201,7 +202,7 @@ func accessAPIClient(address string) (accessproto.AccessAPIClient, io.Closer, er conn, err := grpc.Dial( address, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithInsecure()) //nolint:staticcheck + grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, nil, fmt.Errorf("failed to connect to address %s: %w", address, err) } diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index 2331223c33f..5ef8acdd810 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -20,7 +20,7 @@ import ( // Config defines the configurable options for the ingress server. type Config struct { ListenAddr string - MaxExecutionDataMsgSize int // in bytes + MaxExecutionDataMsgSize uint // in bytes RpcMetricsEnabled bool // enable GRPC metrics } @@ -53,8 +53,8 @@ func NewEng( ) *Engine { // create a GRPC server to serve GRPC clients grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(config.MaxExecutionDataMsgSize), - grpc.MaxSendMsgSize(config.MaxExecutionDataMsgSize), + grpc.MaxRecvMsgSize(int(config.MaxExecutionDataMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxExecutionDataMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors diff --git a/engine/collection/compliance.go b/engine/collection/compliance.go new file mode 100644 index 00000000000..ee03237c533 --- /dev/null +++ b/engine/collection/compliance.go @@ -0,0 +1,27 @@ +package collection + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/component" +) + +// Compliance defines the interface to the cluster consensus logic that precedes hotstuff logic. +// It's responsible for processing incoming block proposals broadcast by other cluster consensus participants +// as well as blocks obtained via the chain sync protocol. +// Compliance logic performs validation of incoming blocks by checking headers and payloads. +// Compliance logic guarantees that only valid blocks are added to chain state, passed to hotstuff and other +// components. +// Implementation need to be non-blocking and concurrency safe. +type Compliance interface { + component.Component + + // OnClusterBlockProposal feeds a new block proposal into the processing pipeline. + // Incoming proposals will be queued and eventually dispatched by worker. + // This method is non-blocking. + OnClusterBlockProposal(proposal flow.Slashable[messages.ClusterBlockProposal]) + // OnSyncedClusterBlock feeds a block obtained from sync proposal into the processing pipeline. + // Incoming proposals will be queued and eventually dispatched by worker. + // This method is non-blocking. + OnSyncedClusterBlock(block flow.Slashable[messages.ClusterBlockProposal]) +} diff --git a/engine/collection/compliance/core.go b/engine/collection/compliance/core.go index d8b07b2dbe0..2edb6faa073 100644 --- a/engine/collection/compliance/core.go +++ b/engine/collection/compliance/core.go @@ -5,17 +5,20 @@ package compliance import ( "errors" "fmt" + "time" "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/state" clusterkv "github.com/onflow/flow-go/state/cluster" @@ -25,20 +28,29 @@ import ( // Core contains the central business logic for the collector clusters' compliance engine. // It is responsible for handling communication for the embedded consensus algorithm. -// NOTE: Core is designed to be non-thread safe and cannot be used in concurrent environment -// user of this object needs to ensure single thread access. +// CAUTION with CONCURRENCY: +// - At the moment, compliance.Core _can not_ process blocks concurrently. Callers of `OnBlockProposal` +// need to ensure single-threaded access. +// - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe +// method of compliance.Core type Core struct { log zerolog.Logger // used to log relevant actions with context config compliance.Config - metrics module.EngineMetrics + engineMetrics module.EngineMetrics mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics collectionMetrics module.CollectionMetrics headers storage.Headers state clusterkv.MutableState + // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks + finalizedView counters.StrictMonotonousCounter + finalizedHeight counters.StrictMonotonousCounter pending module.PendingClusterBlockBuffer // pending block cache sync module.BlockRequester hotstuff module.HotStuff + validator hotstuff.Validator voteAggregator hotstuff.VoteAggregator + timeoutAggregator hotstuff.TimeoutAggregator } // NewCore instantiates the business logic for the collector clusters' compliance engine. @@ -46,11 +58,16 @@ func NewCore( log zerolog.Logger, collector module.EngineMetrics, mempool module.MempoolMetrics, + hotstuffMetrics module.HotstuffMetrics, collectionMetrics module.CollectionMetrics, headers storage.Headers, state clusterkv.MutableState, pending module.PendingClusterBlockBuffer, + sync module.BlockRequester, + validator hotstuff.Validator, + hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, opts ...compliance.Opt, ) (*Core, error) { @@ -62,17 +79,27 @@ func NewCore( c := &Core{ log: log.With().Str("cluster_compliance", "core").Logger(), config: config, - metrics: collector, + engineMetrics: collector, mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, collectionMetrics: collectionMetrics, headers: headers, state: state, pending: pending, - sync: nil, // use `WithSync` - hotstuff: nil, // use `WithConsensus` + sync: sync, + hotstuff: hotstuff, + validator: validator, voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } + // initialize finalized boundary cache + final, err := c.state.Final().Head() + if err != nil { + return nil, fmt.Errorf("could not initialized finalized boundary cache: %w", err) + } + c.ProcessFinalizedBlock(final) + // log the mempool size off the bat c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) @@ -80,39 +107,69 @@ func NewCore( } // OnBlockProposal handles incoming block proposals. +// No errors are expected during normal operation. func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.ClusterBlockProposal) error { + startTime := time.Now() + defer func() { + c.hotstuffMetrics.BlockProcessingDuration(time.Since(startTime)) + }() + block := proposal.Block.ToInternal() header := block.Header + blockID := header.ID() + finalHeight := c.finalizedHeight.Value() + finalView := c.finalizedView.Value() log := c.log.With(). Hex("origin_id", originID[:]). Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). - Hex("block_id", logging.Entity(header)). + Hex("block_id", blockID[:]). Hex("parent_id", header.ParentID[:]). Hex("ref_block_id", block.Payload.ReferenceBlockID[:]). Hex("collection_id", logging.Entity(block.Payload.Collection)). Int("tx_count", block.Payload.Collection.Len()). Time("timestamp", header.Timestamp). Hex("proposer", header.ProposerID[:]). - Hex("signers", header.ParentVoterIndices). + Hex("parent_signer_indices", header.ParentVoterIndices). + Uint64("finalized_height", finalHeight). + Uint64("finalized_view", finalView). Logger() + if log.Debug().Enabled() { + log = log.With().Strs("tx_ids", flow.IdentifierList(block.Payload.Collection.Light().Transactions).Strings()).Logger() + } log.Info().Msg("block proposal received") + // drop proposals below the finalized threshold + if header.Height <= finalHeight || header.View <= finalView { + log.Debug().Msg("dropping block below finalized boundary") + return nil + } + + // ignore proposals which are too far ahead of our local finalized state + // instead, rely on sync engine to catch up finalization more effectively, and avoid + // large subtree of blocks to be cached. + if header.Height > finalHeight+c.config.SkipNewProposalsThreshold { + log.Debug(). + Uint64("skip_new_proposals_threshold", c.config.SkipNewProposalsThreshold). + Msg("dropping block too far ahead of locally finalized height") + return nil + } + // first, we reject all blocks that we don't need to process: // 1) blocks already in the cache; they will already be processed later // 2) blocks already on disk; they were processed and await finalization // ignore proposals that are already cached - _, cached := c.pending.ByID(header.ID()) + _, cached := c.pending.ByID(blockID) if cached { log.Debug().Msg("skipping already cached proposal") return nil } // ignore proposals that were already processed - _, err := c.headers.ByBlockID(header.ID()) + _, err := c.headers.ByBlockID(blockID) if err == nil { log.Debug().Msg("skipping already processed proposal") return nil @@ -121,20 +178,6 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus return fmt.Errorf("could not check proposal: %w", err) } - // ignore proposals which are too far ahead of our local finalized state - // instead, rely on sync engine to catch up finalization more effectively, and avoid - // large subtree of blocks to be cached. - final, err := c.state.Final().Head() - if err != nil { - return fmt.Errorf("could not get latest finalized header: %w", err) - } - if header.Height > final.Height && header.Height-final.Height > c.config.SkipNewProposalsThreshold { - log.Debug(). - Uint64("final_height", final.Height). - Msg("dropping block too far ahead of locally finalized height") - return nil - } - // there are two possibilities if the proposal is neither already pending // processing in the cache, nor has already been processed: // 1) the proposal is unverifiable because the parent is unknown @@ -149,7 +192,6 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus // syncing with range requests for finalized blocks will request for the blocks. _, found := c.pending.ByID(header.ParentID) if found { - // add the block to the cache _ = c.pending.Add(originID, block) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) @@ -160,17 +202,14 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus // if the proposal is connected to a block that is neither in the cache, nor // in persistent storage, its direct parent is missing; cache the proposal // and request the parent - _, err = c.headers.ByBlockID(header.ParentID) + parent, err := c.headers.ByBlockID(header.ParentID) if errors.Is(err, storage.ErrNotFound) { - _ = c.pending.Add(originID, block) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) - log.Debug().Msg("requesting missing parent for proposal") - c.sync.RequestBlock(header.ParentID, header.Height-1) - + log.Debug().Msg("requesting missing parent for proposal") return nil } if err != nil { @@ -183,7 +222,7 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus // execution of the entire recursion, which might include processing the // proposal's pending children. There is another span within // processBlockProposal that measures the time spent for a single proposal. - err = c.processBlockAndDescendants(block) + err = c.processBlockAndDescendants(block, parent) c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) if err != nil { return fmt.Errorf("could not process block proposal: %w", err) @@ -193,36 +232,36 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Clus } // processBlockAndDescendants is a recursive function that processes a block and -// its pending proposals for its children. By induction, any children connected -// to a valid proposal are validly connected to the finalized state and can be +// its pending descendants. By induction, any child block of a +// valid proposal is itself connected to the finalized state and can be // processed as well. -func (c *Core) processBlockAndDescendants(block *cluster.Block) error { - blockID := block.ID() +func (c *Core) processBlockAndDescendants(proposal *cluster.Block, parent *flow.Header) error { + blockID := proposal.ID() + log := c.log.With(). + Str("block_id", blockID.String()). + Uint64("block_height", proposal.Header.Height). + Uint64("block_view", proposal.Header.View). + Uint64("parent_view", parent.View). + Logger() // process block itself - err := c.processBlockProposal(block) - // child is outdated by the time we started processing it - // => node was probably behind and is catching up. Log as warning - if engine.IsOutdatedInputError(err) { - c.log.Info().Msg("dropped processing of abandoned fork; this might be an indicator that the node is slightly behind") - return nil - } - // the block is invalid; log as error as we desire honest participation - // ToDo: potential slashing - if engine.IsInvalidInputError(err) { - c.log.Warn(). - Err(err). - Bool(logging.KeySuspicious, true). - Msg("received invalid block from other node (potential slashing evidence?)") - return nil - } - if engine.IsUnverifiableInputError(err) { - c.log.Warn(). - Err(err). - Msg("received unverifiable from other node") - return nil - } + err := c.processBlockProposal(proposal, parent) if err != nil { + if checkForAndLogOutdatedInputError(err, log) || checkForAndLogUnverifiableInputError(err, log) { + return nil + } + if checkForAndLogInvalidInputError(err, log) { + // notify VoteAggregator about the invalid block + err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(proposal.Header)) + if err != nil { + if mempool.IsBelowPrunedThresholdError(err) { + log.Warn().Msg("received invalid block, but is below pruned threshold") + return nil + } + return fmt.Errorf("unexpected error notifying vote aggregator about invalid block: %w", err) + } + return nil + } // unexpected error: potentially corrupted internal state => abort processing and escalate error return fmt.Errorf("failed to process block %x: %w", blockID, err) } @@ -235,7 +274,7 @@ func (c *Core) processBlockAndDescendants(block *cluster.Block) error { return nil } for _, child := range children { - cpr := c.processBlockAndDescendants(child.Message) + cpr := c.processBlockAndDescendants(child.Message, proposal.Header) if cpr != nil { // unexpected error: potentially corrupted internal state => abort processing and escalate error return cpr @@ -250,13 +289,18 @@ func (c *Core) processBlockAndDescendants(block *cluster.Block) error { // processBlockProposal processes the given block proposal. The proposal must connect to // the finalized state. -func (c *Core) processBlockProposal(block *cluster.Block) error { - header := block.Header +// Expected errors during normal operations: +// - engine.OutdatedInputError if the block proposal is outdated (e.g. orphaned) +// - engine.InvalidInputError if the block proposal is invalid +// - engine.UnverifiableInputError if the proposal cannot be validated +func (c *Core) processBlockProposal(proposal *cluster.Block, parent *flow.Header) error { + header := proposal.Header + blockID := header.ID() log := c.log.With(). Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). - Hex("block_id", logging.Entity(header)). + Hex("block_id", blockID[:]). Hex("parent_id", header.ParentID[:]). Hex("payload_hash", header.PayloadHash[:]). Time("timestamp", header.Timestamp). @@ -265,65 +309,99 @@ func (c *Core) processBlockProposal(block *cluster.Block) error { Logger() log.Info().Msg("processing block proposal") - // see if the block is a valid extension of the protocol state - err := c.state.Extend(block) - // if the block proposes an invalid extension of the protocol state, then the block is invalid - if state.IsInvalidExtensionError(err) { - return engine.NewInvalidInputErrorf("invalid extension of cluster state (block_id: %x, height: %d): %w", - header.ID(), header.Height, err) - } - // protocol state aborted processing of block as it is on an abandoned fork: block is outdated - if state.IsOutdatedExtensionError(err) { - return engine.NewOutdatedInputErrorf("outdated extension of cluster state (block_id: %x, height: %d): %w", - header.ID(), header.Height, err) - } - if state.IsUnverifiableExtensionError(err) { - return engine.NewUnverifiableInputError("unverifiable extension of cluster state (block_id: %x, height: %d): %w", - header.ID(), header.Height, err) - } + hotstuffProposal := model.ProposalFromFlow(header) + err := c.validator.ValidateProposal(hotstuffProposal) if err != nil { - return fmt.Errorf("unexpected error while updating cluster state (block_id: %x, height: %d): %w", header.ID(), header.Height, err) + if model.IsInvalidBlockError(err) { + return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // The cluster committee never returns ErrViewForUnknownEpoch, therefore this case + // is an unexpected error in cluster consensus. + return fmt.Errorf("unexpected error: cluster committee reported unknown epoch : %w", err) + } + return fmt.Errorf("unexpected error validating proposal: %w", err) } - // retrieve the parent - parent, err := c.headers.ByBlockID(header.ParentID) + // see if the block is a valid extension of the protocol state + err = c.state.Extend(proposal) if err != nil { - return fmt.Errorf("could not retrieve proposal parent: %w", err) + if state.IsInvalidExtensionError(err) { + // if the block proposes an invalid extension of the cluster state, then the block is invalid + // TODO: we should slash the block proposer + return engine.NewInvalidInputErrorf("invalid extension of cluster state (block: %x, height: %d): %w", blockID, header.Height, err) + } else if state.IsOutdatedExtensionError(err) { + // cluster state aborted processing of block as it is on an abandoned fork: block is outdated + return engine.NewOutdatedInputErrorf("outdated extension of cluster state: %w", err) + } else if state.IsUnverifiableExtensionError(err) { + return engine.NewUnverifiableInputError("unverifiable extension of cluster state (block_id: %x, height: %d): %w", + header.ID(), header.Height, err) + } else { + // unexpected error: potentially corrupted internal state => abort processing and escalate error + return fmt.Errorf("unexpected exception while extending cluster state with block %x at height %d: %w", blockID, header.Height, err) + } } + // notify vote aggregator about a new block, so that it can start verifying + // votes for it. + c.voteAggregator.AddBlock(hotstuffProposal) + // submit the model to hotstuff for processing + // TODO replace with pubsub https://github.com/dapperlabs/flow-go/issues/6395 log.Info().Msg("forwarding block proposal to hotstuff") - // TODO: wait for the returned callback channel if we are processing blocks from range response - c.hotstuff.SubmitProposal(header, parent.View) - - return nil -} + c.hotstuff.SubmitProposal(hotstuffProposal) -// OnBlockVote handles votes for blocks by passing them to the core consensus -// algorithm -func (c *Core) OnBlockVote(originID flow.Identifier, vote *messages.ClusterBlockVote) error { - - c.log.Debug(). - Hex("origin_id", originID[:]). - Hex("block_id", vote.BlockID[:]). - Uint64("view", vote.View). - Msg("received vote") - - c.voteAggregator.AddVote(&model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, - }) return nil } -// ProcessFinalizedView performs pruning of stale data based on finalization event +// ProcessFinalizedBlock performs pruning of stale data based on finalization event // removes pending blocks below the finalized view -func (c *Core) ProcessFinalizedView(finalizedView uint64) { +func (c *Core) ProcessFinalizedBlock(finalized *flow.Header) { // remove all pending blocks at or below the finalized view - c.pending.PruneByView(finalizedView) + c.pending.PruneByView(finalized.View) + c.finalizedHeight.Set(finalized.Height) + c.finalizedView.Set(finalized.View) // always record the metric c.mempoolMetrics.MempoolEntries(metrics.ResourceClusterProposal, c.pending.Size()) } + +// checkForAndLogOutdatedInputError checks whether error is an `engine.OutdatedInputError`. +// If this is the case, we emit a log message and return true. +// For any error other than `engine.OutdatedInputError`, this function is a no-op +// and returns false. +func checkForAndLogOutdatedInputError(err error, log zerolog.Logger) bool { + if engine.IsOutdatedInputError(err) { + // child is outdated by the time we started processing it + // => node was probably behind and is catching up. Log as warning + log.Info().Msg("dropped processing of abandoned fork; this might be an indicator that the node is slightly behind") + return true + } + return false +} + +// checkForAndLogInvalidInputError checks whether error is an `engine.InvalidInputError`. +// If this is the case, we emit a log message and return true. +// For any error other than `engine.InvalidInputError`, this function is a no-op +// and returns false. +func checkForAndLogInvalidInputError(err error, log zerolog.Logger) bool { + if engine.IsInvalidInputError(err) { + // the block is invalid; log as error as we desire honest participation + log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") + return true + } + return false +} + +// checkForAndLogUnverifiableInputError checks whether error is an `engine.UnverifiableInputError`. +// If this is the case, we emit a log message and return true. +// For any error other than `engine.UnverifiableInputError`, this function is a no-op +// and returns false. +func checkForAndLogUnverifiableInputError(err error, log zerolog.Logger) bool { + if engine.IsUnverifiableInputError(err) { + // the block cannot be validated + log.Err(err).Msg("received unverifiable block proposal; this is an indicator of a proposal that cannot be verified under current state") + return true + } + return false +} diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index 49da68f52b7..c39b5f578c0 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -20,6 +20,7 @@ import ( "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/state" clusterint "github.com/onflow/flow-go/state/cluster" clusterstate "github.com/onflow/flow-go/state/cluster/mock" storerr "github.com/onflow/flow-go/storage" @@ -27,17 +28,17 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func doneChan() <-chan struct{} { - c := make(chan struct{}) - close(c) - return c +func TestComplianceCore(t *testing.T) { + suite.Run(t, new(CoreSuite)) } -func TestComplianceCore(t *testing.T) { - suite.Run(t, new(ComplianceCoreSuite)) +// CoreSuite tests the compliance core logic. +type CoreSuite struct { + CommonSuite } -type ComplianceCoreSuite struct { +// CommonSuite is shared between compliance core and engine testing. +type CommonSuite struct { suite.Suite head *cluster.Block @@ -48,20 +49,22 @@ type ComplianceCoreSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[cluster.Block] // mocked dependencies - state *clusterstate.MutableState - snapshot *clusterstate.Snapshot - metrics *metrics.NoopCollector - headers *storage.Headers - pending *module.PendingClusterBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - voteAggregator *hotstuff.VoteAggregator + state *clusterstate.MutableState + snapshot *clusterstate.Snapshot + metrics *metrics.NoopCollector + headers *storage.Headers + pending *module.PendingClusterBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core } -func (cs *ComplianceCoreSuite) SetupTest() { +func (cs *CommonSuite) SetupTest() { // seed the RNG rand.Seed(time.Now().UnixNano()) @@ -149,9 +152,11 @@ func (cs *ComplianceCoreSuite) SetupTest() { }() // set up hotstuff module mock - cs.hotstuff = &module.HotStuff{} + cs.hotstuff = module.NewHotStuff(cs.T()) - cs.voteAggregator = &hotstuff.VoteAggregator{} + cs.validator = hotstuff.NewValidator(cs.T()) + cs.voteAggregator = hotstuff.NewVoteAggregator(cs.T()) + cs.timeoutAggregator = hotstuff.NewTimeoutAggregator(cs.T()) // set up synchronization module mock cs.sync = &module.BlockRequester{} @@ -167,20 +172,22 @@ func (cs *ComplianceCoreSuite) SetupTest() { cs.metrics, cs.metrics, cs.metrics, + cs.metrics, cs.headers, cs.state, cs.pending, + cs.sync, + cs.validator, + cs.hotstuff, cs.voteAggregator, + cs.timeoutAggregator, ) require.NoError(cs.T(), err, "engine initialization should pass") cs.core = core - // assign engine with consensus & synchronization - cs.core.hotstuff = cs.hotstuff - cs.core.sync = cs.sync } -func (cs *ComplianceCoreSuite) TestOnBlockProposalValidParent() { +func (cs *CoreSuite) TestOnBlockProposalValidParent() { // create a proposal that directly descends from the latest finalized header originID := unittest.IdentifierFixture() @@ -191,17 +198,43 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalValidParent() { // store the data for retrieval cs.headerDB[block.Header.ParentID] = cs.head - cs.hotstuff.On("SubmitProposal", block.Header, cs.head.Header.View).Return(doneChan()) + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error err := cs.core.OnBlockProposal(originID, proposal) require.NoError(cs.T(), err, "valid block proposal should pass") +} - // we should submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) +func (cs *CoreSuite) TestOnBlockProposalValidAncestor() { + + // create a proposal that has two ancestors in the cache + originID := unittest.IdentifierFixture() + ancestor := unittest.ClusterBlockWithParent(cs.head) + parent := unittest.ClusterBlockWithParent(&ancestor) + block := unittest.ClusterBlockWithParent(&parent) + proposal := messages.NewClusterBlockProposal(&block) + + // store the data for retrieval + cs.headerDB[parent.ID()] = &parent + cs.headerDB[ancestor.ID()] = &ancestor + + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() + + // it should be processed without error + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "valid block proposal should pass") + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", &block) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalSkipProposalThreshold() { +func (cs *CoreSuite) TestOnBlockProposalSkipProposalThreshold() { // create a proposal which is far enough ahead to be dropped originID := unittest.IdentifierFixture() @@ -217,7 +250,12 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalSkipProposalThreshold() { cs.pending.AssertNotCalled(cs.T(), "Add", originID, mock.Anything) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalValidAncestor() { +// TestOnBlockProposal_FailsHotStuffValidation tests that a proposal which fails HotStuff validation. +// - should not go through protocol state validation +// - should not be added to the state +// - we should not attempt to process its children +// - we should notify VoteAggregator, for known errors +func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { // create a proposal that has two ancestors in the cache originID := unittest.IdentifierFixture() @@ -225,25 +263,68 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalValidAncestor() { parent := unittest.ClusterBlockWithParent(&ancestor) block := unittest.ClusterBlockWithParent(&parent) proposal := messages.NewClusterBlockProposal(&block) + hotstuffProposal := model.ProposalFromFlow(block.Header) // store the data for retrieval cs.headerDB[parent.ID()] = &parent cs.headerDB[ancestor.ID()] = &ancestor - cs.hotstuff.On("SubmitProposal", block.Header, parent.Header.View).Return(doneChan()) + cs.Run("invalid block error", func() { + // the block fails HotStuff validation + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + // we should notify VoteAggregator about the invalid block + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) - // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) - require.NoError(cs.T(), err, "valid block proposal should pass") + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") - // we should extend the state with the header - cs.state.AssertCalled(cs.T(), "Extend", &block) + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) - // we should submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) + cs.Run("view for unknown epoch error", func() { + // the block fails HotStuff validation + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.ErrViewForUnknownEpoch) + + // this error is not expected should raise an exception + err := cs.core.OnBlockProposal(originID, proposal) + require.Error(cs.T(), err, "proposal with invalid extension should fail") + require.ErrorIs(cs.T(), err, model.ErrViewForUnknownEpoch) + + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("unexpected error", func() { + // the block fails HotStuff validation + unexpectedErr := errors.New("generic unexpected error") + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(unexpectedErr) + + // the error should be propagated + err := cs.core.OnBlockProposal(originID, proposal) + require.ErrorIs(cs.T(), err, unexpectedErr) + + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalInvalidExtension() { +// TestOnBlockProposal_FailsProtocolStateValidation tests processing a proposal which passes HotStuff validation, +// but fails protocol state validation. +// - should not be added to the state +// - we should not attempt to process its children +// - we should notify VoteAggregator, for known errors +func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { // create a proposal that has two ancestors in the cache originID := unittest.IdentifierFixture() @@ -251,32 +332,74 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalInvalidExtension() { parent := unittest.ClusterBlockWithParent(&ancestor) block := unittest.ClusterBlockWithParent(&parent) proposal := messages.NewClusterBlockProposal(&block) + hotstuffProposal := model.ProposalFromFlow(block.Header) // store the data for retrieval cs.headerDB[parent.ID()] = &parent cs.headerDB[ancestor.ID()] = &ancestor - // make sure we fail to extend the state - *cs.state = clusterstate.MutableState{} - cs.state.On("Final").Return( - func() clusterint.Snapshot { - return cs.snapshot - }, - ) - cs.state.On("Extend", mock.Anything).Return(errors.New("dummy error")) + // the block passes HotStuff validation + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) - // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal) - require.Error(cs.T(), err, "proposal with invalid extension should fail") - - // we should extend the state with the header - cs.state.AssertCalled(cs.T(), "Extend", &block) + cs.Run("invalid block", func() { + // make sure we fail to extend the state + *cs.state = clusterstate.MutableState{} + cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) + cs.state.On("Extend", mock.Anything).Return(state.NewInvalidExtensionError("")) + // we should notify VoteAggregator about the invalid block + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) - // we should not submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", &block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("outdated block", func() { + // make sure we fail to extend the state + *cs.state = clusterstate.MutableState{} + cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) + cs.state.On("Extend", mock.Anything).Return(state.NewOutdatedExtensionError("")) + + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", &block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("unexpected error", func() { + // make sure we fail to extend the state + *cs.state = clusterstate.MutableState{} + cs.state.On("Final").Return(func() clusterint.Snapshot { return cs.snapshot }) + unexpectedErr := errors.New("unexpected generic error") + cs.state.On("Extend", mock.Anything).Return(unexpectedErr) + + // it should be processed without error + err := cs.core.OnBlockProposal(originID, proposal) + require.ErrorIs(cs.T(), err, unexpectedErr) + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", &block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything, mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) } -func (cs *ComplianceCoreSuite) TestProcessBlockAndDescendants() { +func (cs *CoreSuite) TestProcessBlockAndDescendants() { // create three children blocks parent := unittest.ClusterBlockWithParent(cs.head) @@ -305,13 +428,15 @@ func (cs *ComplianceCoreSuite) TestProcessBlockAndDescendants() { cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending2) cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending3) - cs.hotstuff.On("SubmitProposal", parent.Header, cs.head.Header.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block1.Header, parent.Header.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block2.Header, parent.Header.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block3.Header, parent.Header.View).Return(doneChan()).Once() + for _, block := range []cluster.Block{parent, block1, block2, block3} { + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() + } // execute the connected children handling - err := cs.core.processBlockAndDescendants(&parent) + err := cs.core.processBlockAndDescendants(&parent, cs.head.Header) require.NoError(cs.T(), err, "should pass handling children") // check that we submitted each child to hotstuff @@ -321,31 +446,7 @@ func (cs *ComplianceCoreSuite) TestProcessBlockAndDescendants() { cs.pending.AssertCalled(cs.T(), "DropForParent", parent.Header.ID()) } -func (cs *ComplianceCoreSuite) TestOnSubmitVote() { - // create a vote - originID := unittest.IdentifierFixture() - vote := messages.ClusterBlockVote{ - BlockID: unittest.IdentifierFixture(), - View: rand.Uint64(), - SigData: unittest.SignatureFixture(), - } - - cs.voteAggregator.On("AddVote", &model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, - }).Return() - - // execute the vote submission - err := cs.core.OnBlockVote(originID, &vote) - require.NoError(cs.T(), err, "block vote should pass") - - // check that submit vote was called with correct parameters - cs.hotstuff.AssertExpectations(cs.T()) -} - -func (cs *ComplianceCoreSuite) TestProposalBufferingOrder() { +func (cs *CoreSuite) TestProposalBufferingOrder() { // create a proposal that we will not submit until the end originID := unittest.IdentifierFixture() @@ -396,14 +497,16 @@ func (cs *ComplianceCoreSuite) TestProposalBufferingOrder() { proposals[1].Header.ID(), proposals[2].Header.ID(), } - cs.hotstuff.On("SubmitProposal", mock.Anything, mock.Anything).Times(4).Run( + cs.hotstuff.On("SubmitProposal", mock.Anything).Times(4).Run( func(args mock.Arguments) { - header := args.Get(0).(*flow.Header) - assert.Equal(cs.T(), order[index], header.ID(), "should submit correct header to hotstuff") + header := args.Get(0).(*model.Proposal).Block + assert.Equal(cs.T(), order[index], header.BlockID, "should submit correct header to hotstuff") index++ - cs.headerDB[header.ID()] = proposalsLookup[header.ID()] + cs.headerDB[header.BlockID] = proposalsLookup[header.BlockID] }, - ).Return(doneChan()) + ) + cs.voteAggregator.On("AddBlock", mock.Anything).Times(4) + cs.validator.On("ValidateProposal", mock.Anything).Times(4).Return(nil) missingProposal := messages.NewClusterBlockProposal(missing) diff --git a/engine/collection/compliance/engine.go b/engine/collection/compliance/engine.go index 3582d262a7a..7705df8f908 100644 --- a/engine/collection/compliance/engine.go +++ b/engine/collection/compliance/engine.go @@ -1,64 +1,50 @@ package compliance import ( - "context" - "errors" "fmt" - "time" "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/tracker" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/engine/common/fifoqueue" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" - "github.com/onflow/flow-go/model/cluster" - "github.com/onflow/flow-go/model/events" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/lifecycle" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/logging" ) -// defaultBlockQueueCapacity maximum capacity of block proposals queue -const defaultBlockQueueCapacity = 10000 - -// defaultVoteQueueCapacity maximum capacity of block votes queue -const defaultVoteQueueCapacity = 1000 +// defaultBlockQueueCapacity maximum capacity of inbound queue for `messages.ClusterBlockProposal`s +const defaultBlockQueueCapacity = 10_000 // Engine is a wrapper struct for `Core` which implements cluster consensus algorithm. // Engine is responsible for handling incoming messages, queueing for processing, broadcasting proposals. +// Implements collection.Compliance interface. type Engine struct { - unit *engine.Unit - lm *lifecycle.LifecycleManager - log zerolog.Logger - metrics module.EngineMetrics - me module.Local - headers storage.Headers - payloads storage.ClusterPayloads - state protocol.State - core *Core - pendingBlocks engine.MessageStore - pendingVotes engine.MessageStore - messageHandler *engine.MessageHandler - finalizedView counters.StrictMonotonousCounter - finalizationEventsNotifier engine.Notifier - con network.Conduit - stopHotstuff context.CancelFunc - cluster flow.IdentityList // consensus participants in our cluster -} + *component.ComponentManager + log zerolog.Logger + metrics module.EngineMetrics + me module.Local + headers storage.Headers + payloads storage.ClusterPayloads + state protocol.State + core *Core + pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks + pendingBlocksNotifier engine.Notifier + finalizedBlockTracker *tracker.NewestBlockTracker + finalizedBlockNotifier engine.Notifier +} + +var _ collection.Compliance = (*Engine)(nil) func NewEngine( log zerolog.Logger, - net network.Network, me module.Local, state protocol.State, payloads storage.ClusterPayloads, @@ -66,17 +52,6 @@ func NewEngine( ) (*Engine, error) { engineLog := log.With().Str("cluster_compliance", "engine").Logger() - // find my cluster for the current epoch - // TODO this should flow from cluster state as source of truth - clusters, err := state.Final().Epochs().Current().Clustering() - if err != nil { - return nil, fmt.Errorf("could not get clusters: %w", err) - } - currentCluster, _, found := clusters.ByNodeID(me.NodeID()) - if !found { - return nil, fmt.Errorf("could not find cluster for self") - } - // FIFO queue for block proposals blocksQueue, err := fifoqueue.NewFifoQueue( defaultBlockQueueCapacity, @@ -85,224 +60,69 @@ func NewEngine( }), ) if err != nil { - return nil, fmt.Errorf("failed to create queue for inbound receipts: %w", err) - } - pendingBlocks := &engine.FifoMessageStore{ - FifoQueue: blocksQueue, - } - - // FIFO queue for block votes - votesQueue, err := fifoqueue.NewFifoQueue( - defaultVoteQueueCapacity, - fifoqueue.WithLengthObserver(func(len int) { core.mempoolMetrics.MempoolEntries(metrics.ResourceClusterBlockVoteQueue, uint(len)) }), - ) - if err != nil { - return nil, fmt.Errorf("failed to create queue for inbound approvals: %w", err) + return nil, fmt.Errorf("failed to create queue for inbound block proposals: %w", err) } - pendingVotes := &engine.FifoMessageStore{FifoQueue: votesQueue} - - // define message queueing behaviour - handler := engine.NewMessageHandler( - engineLog, - engine.NewNotifier(), - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*messages.ClusterBlockProposal) - if ok { - core.metrics.MessageReceived(metrics.EngineClusterCompliance, metrics.MessageClusterBlockProposal) - } - return ok - }, - Store: pendingBlocks, - }, - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*events.SyncedClusterBlock) - if ok { - core.metrics.MessageReceived(metrics.EngineClusterCompliance, metrics.MessageSyncedClusterBlock) - } - return ok - }, - Map: func(msg *engine.Message) (*engine.Message, bool) { - syncedBlock := msg.Payload.(*events.SyncedClusterBlock) - msg = &engine.Message{ - OriginID: msg.OriginID, - Payload: &messages.ClusterBlockProposal{ - Block: syncedBlock.Block, - }, - } - return msg, true - }, - Store: pendingBlocks, - }, - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*messages.ClusterBlockVote) - if ok { - core.metrics.MessageReceived(metrics.EngineClusterCompliance, metrics.MessageClusterBlockVote) - } - return ok - }, - Store: pendingVotes, - }, - ) eng := &Engine{ - unit: engine.NewUnit(), - lm: lifecycle.NewLifecycleManager(), - log: engineLog, - metrics: core.metrics, - me: me, - headers: core.headers, - payloads: payloads, - state: state, - core: core, - pendingBlocks: pendingBlocks, - pendingVotes: pendingVotes, - messageHandler: handler, - finalizationEventsNotifier: engine.NewNotifier(), - con: nil, - cluster: currentCluster, - } - - chainID, err := core.state.Params().ChainID() - if err != nil { - return nil, fmt.Errorf("could not get chain ID: %w", err) - } - - // register network conduit - conduit, err := net.Register(channels.ConsensusCluster(chainID), eng) - if err != nil { - return nil, fmt.Errorf("could not register engine: %w", err) - } - eng.con = conduit + log: engineLog, + metrics: core.engineMetrics, + me: me, + headers: core.headers, + payloads: payloads, + state: state, + core: core, + pendingBlocks: blocksQueue, + pendingBlocksNotifier: engine.NewNotifier(), + finalizedBlockTracker: tracker.NewNewestBlockTracker(), + finalizedBlockNotifier: engine.NewNotifier(), + } + + // create the component manager and worker threads + eng.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(eng.processBlocksLoop). + AddWorker(eng.finalizationProcessingLoop). + Build() return eng, nil } -// WithConsensus adds the consensus algorithm to the engine. This must be -// called before the engine can start. -func (e *Engine) WithConsensus(hot module.HotStuff) *Engine { - e.core.hotstuff = hot - return e -} - -// WithSync adds the block requester to the engine. This must be -// called before the engine can start. -func (e *Engine) WithSync(sync module.BlockRequester) *Engine { - e.core.sync = sync - return e -} - -// Ready returns a ready channel that is closed once the engine has fully -// started. For consensus engine, this is true once the underlying consensus -// algorithm has started. -func (e *Engine) Ready() <-chan struct{} { - if e.core.hotstuff == nil { - panic("must initialize compliance engine with hotstuff engine") - } - e.lm.OnStart(func() { - e.unit.Launch(e.loop) - e.unit.Launch(e.finalizationProcessingLoop) - - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, hotstuffErrChan := irrecoverable.WithSignaler(ctx) - e.stopHotstuff = cancel - - // TODO: this workaround for handling fatal HotStuff errors is required only - // because this engine and epochmgr do not use the Component pattern yet - e.unit.Launch(func() { - e.handleHotStuffError(hotstuffErrChan) - }) - - e.core.hotstuff.Start(signalerCtx) - // wait for request handler to startup - <-e.core.hotstuff.Ready() - }) - return e.lm.Started() -} - -// Done returns a done channel that is closed once the engine has fully stopped. -// For the consensus engine, we wait for hotstuff to finish. -func (e *Engine) Done() <-chan struct{} { - e.lm.OnStop(func() { - e.log.Info().Msg("shutting down hotstuff eventloop") - e.stopHotstuff() - <-e.core.hotstuff.Done() - e.log.Info().Msg("all components have been shut down") - <-e.unit.Done() - }) - return e.lm.Stopped() -} - -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - err := e.ProcessLocal(event) - if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// Submit submits the given event from the node with the given origin ID -// for processing in a non-blocking manner. It returns instantly and logs -// a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { - err := e.Process(channel, originID, event) - if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// ProcessLocal processes an event originating on the local node. -func (e *Engine) ProcessLocal(event interface{}) error { - return e.messageHandler.Process(e.me.NodeID(), event) -} +// processBlocksLoop processes available blocks as they are queued. +func (e *Engine) processBlocksLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() -// Process processes the given event from the node with the given origin ID in -// a blocking manner. It returns the potential processing error when done. -func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - err := e.messageHandler.Process(originID, event) - if err != nil { - if engine.IsIncompatibleInputTypeError(err) { - e.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, event, channel) - return nil - } - return fmt.Errorf("unexpected error while processing engine message: %w", err) - } - return nil -} - -func (e *Engine) loop() { + doneSignal := ctx.Done() + newMessageSignal := e.pendingBlocksNotifier.Channel() for { select { - case <-e.unit.Quit(): + case <-doneSignal: return - case <-e.messageHandler.GetNotifier(): - err := e.processAvailableMessages() + case <-newMessageSignal: + err := e.processQueuedBlocks(doneSignal) if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing queued message") + ctx.Throw(err) } } } } -func (e *Engine) processAvailableMessages() error { - +// processQueuedBlocks processes any available messages from the inbound queues. +// Only returns when all inbound queues are empty (or the engine is terminated). +// No errors expected during normal operations. +func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { for { - msg, ok := e.pendingBlocks.Get() - if ok { - err := e.core.OnBlockProposal(msg.OriginID, msg.Payload.(*messages.ClusterBlockProposal)) - if err != nil { - return fmt.Errorf("could not handle block proposal: %w", err) - } - continue + select { + case <-doneSignal: + return nil + default: } - msg, ok = e.pendingVotes.Get() + msg, ok := e.pendingBlocks.Pop() if ok { - err := e.core.OnBlockVote(msg.OriginID, msg.Payload.(*messages.ClusterBlockVote)) + inBlock := msg.(flow.Slashable[messages.ClusterBlockProposal]) + err := e.core.OnBlockProposal(inBlock.OriginID, inBlock.Message) + e.core.engineMetrics.MessageHandled(metrics.EngineClusterCompliance, metrics.MessageBlockProposal) if err != nil { - return fmt.Errorf("could not handle block vote: %w", err) + return fmt.Errorf("could not handle block proposal: %w", err) } continue } @@ -313,163 +133,56 @@ func (e *Engine) processAvailableMessages() error { } } -// SendVote will send a vote to the desired node. -func (e *Engine) SendVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) error { - - log := e.log.With(). - Hex("collection_id", blockID[:]). - Uint64("collection_view", view). - Hex("recipient_id", recipientID[:]). - Logger() - log.Info().Msg("processing vote transmission request from hotstuff") - - // build the vote message - vote := &messages.ClusterBlockVote{ - BlockID: blockID, - View: view, - SigData: sigData, - } - - // TODO: this is a hot-fix to mitigate the effects of the following Unicast call blocking occasionally - e.unit.Launch(func() { - // send the vote the desired recipient - err := e.con.Unicast(vote, recipientID) - if err != nil { - log.Warn().Err(err).Msg("could not send vote") - return - } - e.metrics.MessageSent(metrics.EngineClusterCompliance, metrics.MessageClusterBlockVote) - log.Info().Msg("collection vote transmitted") - }) - - return nil -} - -// BroadcastProposalWithDelay submits a cluster block proposal (effectively a proposal -// for the next collection) to all the collection nodes in our cluster. -func (e *Engine) BroadcastProposalWithDelay(header *flow.Header, delay time.Duration) error { - - // first, check that we are the proposer of the block - if header.ProposerID != e.me.NodeID() { - return fmt.Errorf("cannot broadcast proposal with non-local proposer (%x)", header.ProposerID) - } - - // get the parent of the block - parent, err := e.headers.ByBlockID(header.ParentID) - if err != nil { - return fmt.Errorf("could not retrieve proposal parent: %w", err) - } - - // fill in the fields that can't be populated by HotStuff - //TODO clean this up - currently we set these fields in builder, then lose - // them in HotStuff, then need to set them again here - header.ChainID = parent.ChainID - header.Height = parent.Height + 1 - - // retrieve the payload for the block - payload, err := e.payloads.ByBlockID(header.ID()) - if err != nil { - return fmt.Errorf("could not get payload for block: %w", err) - } - - log := e.log.With(). - Str("chain_id", header.ChainID.String()). - Uint64("block_height", header.Height). - Uint64("block_view", header.View). - Hex("block_id", logging.ID(header.ID())). - Hex("parent_id", header.ParentID[:]). - Hex("ref_block", payload.ReferenceBlockID[:]). - Int("transaction_count", payload.Collection.Len()). - Hex("parent_signer_indices", header.ParentVoterIndices). - Dur("delay", delay). - Logger() - - log.Debug().Msg("processing cluster broadcast request from hotstuff") - - // retrieve all collection nodes in our cluster - recipients, err := e.state.Final().Identities(filter.And( - filter.In(e.cluster), - filter.Not(filter.HasNodeID(e.me.NodeID())), - )) - if err != nil { - return fmt.Errorf("could not get cluster members: %w", err) - } - - e.unit.LaunchAfter(delay, func() { - - go e.core.hotstuff.SubmitProposal(header, parent.View) - - // create the proposal message for the collection - block := &cluster.Block{ - Header: header, - Payload: payload, - } - msg := messages.NewClusterBlockProposal(block) - - err := e.con.Publish(msg, recipients.NodeIDs()...) - if errors.Is(err, network.EmptyTargetList) { - return - } - if err != nil { - log.Error().Err(err).Msg("could not broadcast proposal") - return - } - - log.Info().Msg("cluster proposal proposed") - - e.metrics.MessageSent(metrics.EngineClusterCompliance, metrics.MessageClusterBlockProposal) - e.core.collectionMetrics.ClusterBlockProposed(block) - }) - - return nil -} - -// BroadcastProposal will propagate a block proposal to all non-local consensus nodes. -// Note the header has incomplete fields, because it was converted from a hotstuff. -func (e *Engine) BroadcastProposal(header *flow.Header) error { - return e.BroadcastProposalWithDelay(header, 0) -} - // OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// -// (1) Informs sealing.Core about finalization of respective block. +// It informs compliance.Core about finalization of the respective block. // // CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages // from external nodes cannot be considered as inputs to this function func (e *Engine) OnFinalizedBlock(block *model.Block) { - if e.finalizedView.Set(block.View) { - e.finalizationEventsNotifier.Notify() + if e.finalizedBlockTracker.Track(block) { + e.finalizedBlockNotifier.Notify() } } -// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events -func (e *Engine) finalizationProcessingLoop() { - finalizationNotifier := e.finalizationEventsNotifier.Channel() - for { - select { - case <-e.unit.Quit(): - return - case <-finalizationNotifier: - e.core.ProcessFinalizedView(e.finalizedView.Value()) - } +// OnClusterBlockProposal feeds a new block proposal into the processing pipeline. +// Incoming proposals are queued and eventually dispatched by worker. +func (e *Engine) OnClusterBlockProposal(proposal flow.Slashable[messages.ClusterBlockProposal]) { + e.core.engineMetrics.MessageReceived(metrics.EngineClusterCompliance, metrics.MessageBlockProposal) + if e.pendingBlocks.Push(proposal) { + e.pendingBlocksNotifier.Notify() + } else { + e.core.engineMetrics.InboundMessageDropped(metrics.EngineClusterCompliance, metrics.MessageBlockProposal) } } -// handleHotStuffError accepts the error channel from the HotStuff component and -// crashes the node if any error is detected. -// -// TODO: this function should be removed in favour of refactoring this engine and -// the epochmgr engine to use the Component pattern, so that irrecoverable errors -// can be bubbled all the way to the node scaffold -func (e *Engine) handleHotStuffError(hotstuffErrs <-chan error) { +// OnSyncedClusterBlock feeds a block obtained from sync proposal into the processing pipeline. +// Incoming proposals are queued and eventually dispatched by worker. +func (e *Engine) OnSyncedClusterBlock(syncedBlock flow.Slashable[messages.ClusterBlockProposal]) { + e.core.engineMetrics.MessageReceived(metrics.EngineClusterCompliance, metrics.MessageSyncedClusterBlock) + if e.pendingBlocks.Push(syncedBlock) { + e.pendingBlocksNotifier.Notify() + } else { + e.core.engineMetrics.InboundMessageDropped(metrics.EngineClusterCompliance, metrics.MessageSyncedClusterBlock) + } +} + +// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events +func (e *Engine) finalizationProcessingLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + doneSignal := ctx.Done() + blockFinalizedSignal := e.finalizedBlockNotifier.Channel() for { select { - case <-e.unit.Quit(): + case <-doneSignal: return - case err := <-hotstuffErrs: - if err != nil { - e.log.Fatal().Err(err).Msg("encountered fatal error in HotStuff") + case <-blockFinalizedSignal: + // retrieve the latest finalized header, so we know the height + finalHeader, err := e.headers.ByBlockID(e.finalizedBlockTracker.NewestBlock().BlockID) + if err != nil { // no expected errors + ctx.Throw(err) } + e.core.ProcessFinalizedBlock(finalHeader) } } } diff --git a/engine/collection/compliance/engine_test.go b/engine/collection/compliance/engine_test.go index f0a4061f905..6df4a1cc03a 100644 --- a/engine/collection/compliance/engine_test.go +++ b/engine/collection/compliance/engine_test.go @@ -1,20 +1,21 @@ package compliance import ( - "math/rand" + "context" "sync" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/irrecoverable" module "github.com/onflow/flow-go/module/mock" netint "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" @@ -26,11 +27,11 @@ import ( ) func TestComplianceEngine(t *testing.T) { - suite.Run(t, new(ComplianceSuite)) + suite.Run(t, new(EngineSuite)) } -type ComplianceSuite struct { - ComplianceCoreSuite +type EngineSuite struct { + CommonSuite clusterID flow.ChainID myID flow.Identifier @@ -43,13 +44,17 @@ type ComplianceSuite struct { payloadDB map[flow.Identifier]*cluster.Payload + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error + engine *Engine } -func (cs *ComplianceSuite) SetupTest() { - cs.ComplianceCoreSuite.SetupTest() +func (cs *EngineSuite) SetupTest() { + cs.CommonSuite.SetupTest() - // initialize the paramaters + // initialize the parameters cs.cluster = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection), unittest.WithWeight(1000), @@ -128,197 +133,97 @@ func (cs *ComplianceSuite) SetupTest() { nil, ) - e, err := NewEngine(unittest.Logger(), cs.net, cs.me, cs.protoState, cs.payloads, cs.core) + e, err := NewEngine(unittest.Logger(), cs.me, cs.protoState, cs.payloads, cs.core) require.NoError(cs.T(), err) cs.engine = e - ready := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() + cs.ctx, cs.cancel, cs.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + cs.engine.Start(cs.ctx) + go unittest.FailOnIrrecoverableError(cs.T(), cs.ctx.Done(), cs.errs) - cs.hotstuff.On("Start", mock.Anything) - cs.hotstuff.On("Ready", mock.Anything).Return(ready) - <-cs.engine.Ready() + unittest.AssertClosesBefore(cs.T(), cs.engine.Ready(), time.Second) } -// TestSendVote tests that single vote can be send and properly processed -func (cs *ComplianceSuite) TestSendVote() { - // create parameters to send a vote - blockID := unittest.IdentifierFixture() - view := rand.Uint64() - sig := unittest.SignatureFixture() - recipientID := unittest.IdentifierFixture() - - // submit the vote - err := cs.engine.SendVote(blockID, view, sig, recipientID) - require.NoError(cs.T(), err, "should pass send vote") - - done := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - cs.hotstuff.On("Done", mock.Anything).Return(done) - - // The vote is transmitted asynchronously. We allow 10ms for the vote to be received: - <-time.After(10 * time.Millisecond) - <-cs.engine.Done() - - // check it was called with right params - vote := messages.ClusterBlockVote{ - BlockID: blockID, - View: view, - SigData: sig, +// TearDownTest stops the engine and checks there are no errors thrown to the SignallerContext. +func (cs *EngineSuite) TearDownTest() { + cs.cancel() + unittest.RequireCloseBefore(cs.T(), cs.engine.Done(), time.Second, "engine failed to stop") + select { + case err := <-cs.errs: + assert.NoError(cs.T(), err) + default: } - cs.con.AssertCalled(cs.T(), "Unicast", &vote, recipientID) -} - -// TestBroadcastProposalWithDelay tests broadcasting proposals with different -// inputs -func (cs *ComplianceSuite) TestBroadcastProposalWithDelay() { - - // generate a parent with height and chain ID set - parent := unittest.ClusterBlockFixture() - parent.Header.ChainID = "test" - parent.Header.Height = 10 - cs.headerDB[parent.ID()] = &parent - - // create a block with the parent and store the payload with correct ID - block := unittest.ClusterBlockWithParent(&parent) - block.Header.ProposerID = cs.myID - cs.payloadDB[block.ID()] = block.Payload - - // keep a duplicate of the correct header to check against leader - header := block.Header - - // unset chain and height to make sure they are correctly reconstructed - block.Header.ChainID = "" - block.Header.Height = 0 - - cs.hotstuff.On("SubmitProposal", block.Header, parent.Header.View).Return(doneChan()).Once() - - // submit to broadcast proposal - err := cs.engine.BroadcastProposalWithDelay(block.Header, 0) - require.NoError(cs.T(), err, "header broadcast should pass") - - // make sure chain ID and height were reconstructed and - // we broadcast to correct nodes - header.ChainID = "test" - header.Height = 11 - msg := messages.NewClusterBlockProposal(&block) - - done := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - cs.hotstuff.On("Done", mock.Anything).Return(done) - - <-time.After(10 * time.Millisecond) - <-cs.engine.Done() - cs.con.AssertCalled(cs.T(), "Publish", msg, cs.cluster[1].NodeID, cs.cluster[2].NodeID) - - // should fail with wrong proposer - header.ProposerID = unittest.IdentifierFixture() - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with wrong proposer") - header.ProposerID = cs.myID - - // should fail with changed (missing) parent - header.ParentID[0]++ - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with missing parent") - header.ParentID[0]-- - - // should fail with wrong block ID (payload unavailable) - header.View++ - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with missing payload") - header.View-- } // TestSubmittingMultipleVotes tests that we can send multiple votes and they // are queued and processed in expected way -func (cs *ComplianceSuite) TestSubmittingMultipleEntries() { - // create a vote - originID := unittest.IdentifierFixture() - voteCount := 15 - - channel := channels.ConsensusCluster(cs.clusterID) - +func (cs *EngineSuite) TestSubmittingMultipleEntries() { + blockCount := 15 var wg sync.WaitGroup wg.Add(1) go func() { - for i := 0; i < voteCount; i++ { - vote := messages.ClusterBlockVote{ - BlockID: unittest.IdentifierFixture(), - View: rand.Uint64(), - SigData: unittest.SignatureFixture(), - } - cs.voteAggregator.On("AddVote", &model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, - }).Return().Once() - // execute the vote submission - _ = cs.engine.Process(channel, originID, &vote) + for i := 0; i < blockCount; i++ { + block := unittest.ClusterBlockWithParent(cs.head) + proposal := messages.NewClusterBlockProposal(&block) + // store the data for retrieval + cs.headerDB[block.Header.ParentID] = cs.head + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil).Once() + // execute the block submission + cs.engine.OnClusterBlockProposal(flow.Slashable[messages.ClusterBlockProposal]{ + OriginID: unittest.IdentifierFixture(), + Message: proposal, + }) } wg.Done() }() wg.Add(1) go func() { // create a proposal that directly descends from the latest finalized header - originID := cs.cluster[1].NodeID block := unittest.ClusterBlockWithParent(cs.head) proposal := messages.NewClusterBlockProposal(&block) // store the data for retrieval cs.headerDB[block.Header.ParentID] = cs.head - cs.hotstuff.On("SubmitProposal", block.Header, cs.head.Header.View).Return(doneChan()) - _ = cs.engine.Process(channel, originID, proposal) + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil).Once() + cs.engine.OnClusterBlockProposal(flow.Slashable[messages.ClusterBlockProposal]{ + OriginID: unittest.IdentifierFixture(), + Message: proposal, + }) wg.Done() }() wg.Wait() - time.Sleep(time.Second) - - // check that submit vote was called with correct parameters - cs.hotstuff.AssertExpectations(cs.T()) - cs.voteAggregator.AssertExpectations(cs.T()) -} - -// TestProcessUnsupportedMessageType tests that Process and ProcessLocal correctly handle a case where invalid message type -// was submitted from network layer. -func (cs *ComplianceSuite) TestProcessUnsupportedMessageType() { - invalidEvent := uint64(42) - err := cs.engine.Process("ch", unittest.IdentifierFixture(), invalidEvent) - // shouldn't result in error since byzantine inputs are expected - require.NoError(cs.T(), err) - // in case of local processing error cannot be consumed since all inputs are trusted - err = cs.engine.ProcessLocal(invalidEvent) - require.Error(cs.T(), err) - require.True(cs.T(), engine.IsIncompatibleInputTypeError(err)) + // wait for the votes queue to drain + assert.Eventually(cs.T(), func() bool { + return cs.engine.pendingBlocks.Len() == 0 + }, time.Second, time.Millisecond*10) } // TestOnFinalizedBlock tests if finalized block gets processed when send through `Engine`. // Tests the whole processing pipeline. -func (cs *ComplianceSuite) TestOnFinalizedBlock() { +func (cs *EngineSuite) TestOnFinalizedBlock() { finalizedBlock := unittest.ClusterBlockFixture() cs.head = &finalizedBlock + cs.headerDB[finalizedBlock.ID()] = &finalizedBlock *cs.pending = module.PendingClusterBlockBuffer{} - cs.pending.On("PruneByView", finalizedBlock.Header.View).Return(nil).Once() - cs.pending.On("Size").Return(uint(0)).Once() - cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock.Header, finalizedBlock.Header.View-1)) - - require.Eventually(cs.T(), - func() bool { - return cs.pending.AssertCalled(cs.T(), "PruneByView", finalizedBlock.Header.View) - }, time.Second, time.Millisecond*20) + // wait for both expected calls before ending the test + wg := new(sync.WaitGroup) + wg.Add(2) + cs.pending.On("PruneByView", finalizedBlock.Header.View). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil).Once() + cs.pending.On("Size"). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(uint(0)).Once() + + cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock.Header)) + unittest.AssertReturnsBefore(cs.T(), wg.Wait, time.Second, "an expected call to block buffer wasn't made") } diff --git a/engine/collection/epochmgr/engine.go b/engine/collection/epochmgr/engine.go index 8bb06a18bc5..40387d1bddd 100644 --- a/engine/collection/epochmgr/engine.go +++ b/engine/collection/epochmgr/engine.go @@ -1,264 +1,275 @@ package epochmgr import ( - "context" "errors" "fmt" + "sync" "time" "github.com/rs/zerolog" - "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/epochs" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/mempool/epochs" + epochpool "github.com/onflow/flow-go/module/mempool/epochs" "github.com/onflow/flow-go/module/util" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/events" ) -// DefaultStartupTimeout is the default time we wait when starting epoch -// components before giving up. -const DefaultStartupTimeout = 30 * time.Second +// DefaultStartupTimeout is the default time we wait when starting epoch components before giving up. +const DefaultStartupTimeout = time.Minute // ErrNotAuthorizedForEpoch is returned when we attempt to create epoch components // for an epoch in which we are not an authorized network participant. This is the // case for epochs during which this node is joining or leaving the network. var ErrNotAuthorizedForEpoch = fmt.Errorf("we are not an authorized participant for the epoch") -// EpochComponents represents all dependencies for running an epoch. -type EpochComponents struct { - *component.ComponentManager - state cluster.State - prop network.Engine - sync network.Engine - hotstuff module.HotStuff - aggregator hotstuff.VoteAggregator -} - -var _ component.Component = (*EpochComponents)(nil) - -func NewEpochComponents( - state cluster.State, - prop network.Engine, - sync network.Engine, - hotstuff module.HotStuff, - aggregator hotstuff.VoteAggregator, -) *EpochComponents { - components := &EpochComponents{ - state: state, - prop: prop, - sync: sync, - hotstuff: hotstuff, - aggregator: aggregator, - } - - builder := component.NewComponentManagerBuilder() - // start new worker that will start child components and wait for them to finish - builder.AddWorker(func(parentCtx irrecoverable.SignalerContext, ready component.ReadyFunc) { - // create a separate context that is not connected to parent, reason: - // we want to stop vote aggregator after event loop and compliance engine have shutdown - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - // start aggregator, hotstuff will be started by compliance engine - aggregator.Start(signalerCtx) - // wait until all components start - <-util.AllReady(components.prop, components.sync, components.aggregator) - // signal that startup has finished and we are ready to go - ready() - // wait for shutdown to be commenced - <-parentCtx.Done() - // wait for compliance engine and event loop to shut down - <-util.AllDone(components.prop, components.sync) - // after event loop and engines were stopped proceed with stopping vote aggregator - cancel() - // wait until it stops - <-components.aggregator.Done() - }) - components.ComponentManager = builder.Build() - - return components -} - -type StartableEpochComponents struct { - *EpochComponents - signalerCtx irrecoverable.SignalerContext // used to start the component - cancel context.CancelFunc // used to stop the epoch components -} - -func NewStartableEpochComponents(components *EpochComponents, signalerCtx irrecoverable.SignalerContext, cancel context.CancelFunc) *StartableEpochComponents { - return &StartableEpochComponents{ - EpochComponents: components, - signalerCtx: signalerCtx, - cancel: cancel, - } -} - // Engine is the epoch manager, which coordinates the lifecycle of other modules // and processes that are epoch-dependent. The manager is responsible for // spinning up engines when a new epoch is about to start and spinning down // engines for an epoch that has ended. +// +// The `epochmgr.Engine` implements the `protocol.Consumer` interface. In particular, it +// ingests the following notifications from the protocol state: +// - EpochSetupPhaseStarted +// - EpochTransition +// +// As part of the engine starting, it executes pending actions that should have been triggered +// by protocol events but those events were missed during a crash/restart. See respective +// consumer methods for further details. type Engine struct { events.Noop // satisfy protocol events consumer interface - unit *engine.Unit - log zerolog.Logger - me module.Local - state protocol.State - pools *epochs.TransactionPools // epoch-scoped transaction pools - factory EpochComponentsFactory // consolidates creating epoch for an epoch - voter module.ClusterRootQCVoter // manages process of voting for next epoch's QC - heightEvents events.Heights // allows subscribing to particular heights - irrecoverableCtx irrecoverable.SignalerContext // parent context for canceling all started epochs - stopComponents context.CancelFunc // used to stop all components - - epochs map[uint64]*StartableEpochComponents // epoch-scoped components per epoch - startupTimeout time.Duration // how long we wait for epoch components to start up + log zerolog.Logger + me module.Local + state protocol.State + pools *epochpool.TransactionPools // epoch-scoped transaction pools + factory EpochComponentsFactory // consolidates creating epoch for an epoch + voter module.ClusterRootQCVoter // manages process of voting for next epoch's QC + heightEvents events.Heights // allows subscribing to particular heights + startupTimeout time.Duration // how long we wait for epoch components to start up + + mu sync.RWMutex // protects epochs map + epochs map[uint64]*RunningEpochComponents // epoch-scoped components per epoch + + // internal event notifications + epochTransitionEvents chan *flow.Header // sends first block of new epoch + epochSetupPhaseStartedEvents chan *flow.Header // sends first block of EpochSetup phase + epochStopEvents chan uint64 // sends counter of epoch to stop + + cm *component.ComponentManager + component.Component } +var _ component.Component = (*Engine)(nil) +var _ protocol.Consumer = (*Engine)(nil) + func New( log zerolog.Logger, me module.Local, state protocol.State, - pools *epochs.TransactionPools, + pools *epochpool.TransactionPools, voter module.ClusterRootQCVoter, factory EpochComponentsFactory, heightEvents events.Heights, ) (*Engine, error) { - ctx, stopComponents := context.WithCancel(context.Background()) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) - e := &Engine{ - unit: engine.NewUnit(), - log: log.With().Str("engine", "epochmgr").Logger(), - me: me, - state: state, - pools: pools, - voter: voter, - factory: factory, - heightEvents: heightEvents, - epochs: make(map[uint64]*StartableEpochComponents), - startupTimeout: DefaultStartupTimeout, - irrecoverableCtx: signalerCtx, - stopComponents: stopComponents, + log: log.With().Str("engine", "epochmgr").Logger(), + me: me, + state: state, + pools: pools, + voter: voter, + factory: factory, + heightEvents: heightEvents, + epochs: make(map[uint64]*RunningEpochComponents), + startupTimeout: DefaultStartupTimeout, + epochTransitionEvents: make(chan *flow.Header, 1), + epochSetupPhaseStartedEvents: make(chan *flow.Header, 1), + epochStopEvents: make(chan uint64, 1), } - // set up epoch-scoped epoch managed by this engine for the current epoch - epoch := e.state.Final().Epochs().Current() - counter, err := epoch.Counter() + e.cm = component.NewComponentManagerBuilder(). + AddWorker(e.handleEpochEvents). + Build() + e.Component = e.cm + + return e, nil +} + +// Start starts the engine. +func (e *Engine) Start(ctx irrecoverable.SignalerContext) { + // (1) start engine-scoped workers + e.cm.Start(ctx) + + // (2) Retrieve protocol state as of latest finalized block. We use this state + // to catch up on events, whose execution was missed during crash-restart. + finalSnapshot := e.state.Final() + currentEpoch := finalSnapshot.Epochs().Current() + currentEpochCounter, err := currentEpoch.Counter() if err != nil { - return nil, fmt.Errorf("could not get epoch counter: %w", err) + ctx.Throw(fmt.Errorf("could not get epoch counter: %w", err)) } - components, err := e.createEpochComponents(epoch) - // don't set up consensus components if we aren't authorized in current epoch - if errors.Is(err, ErrNotAuthorizedForEpoch) { - return e, nil - } + // (3) check if we should attempt to vote after startup + err = e.checkShouldVoteOnStartup(finalSnapshot) if err != nil { - return nil, fmt.Errorf("could not create epoch components for current epoch: %w", err) + ctx.Throw(fmt.Errorf("could not vote on startup: %w", err)) } - ctx, cancel := context.WithCancel(e.irrecoverableCtx) - signalerCtx, _ = irrecoverable.WithSignaler(ctx) - - e.epochs[counter] = NewStartableEpochComponents(components, signalerCtx, cancel) + // (4) start epoch-scoped components: + // set up epoch-scoped epoch managed by this engine for the current epoch + components, err := e.createEpochComponents(currentEpoch) + if err != nil { + if errors.Is(err, ErrNotAuthorizedForEpoch) { + // don't set up consensus components if we aren't authorized in current epoch + e.log.Info().Msg("node is not authorized for current epoch - skipping initializing cluster consensus") + return + } + ctx.Throw(fmt.Errorf("could not create epoch components: %w", err)) + } + err = e.startEpochComponents(ctx, currentEpochCounter, components) + if err != nil { + // all failures to start epoch components are critical + ctx.Throw(fmt.Errorf("could not start epoch components: %w", err)) + } - return e, nil + // TODO if we are within the first 600 blocks of an epoch, we should resume the previous epoch's cluster consensus here https://github.com/dapperlabs/flow-go/issues/5659 } -// Ready returns a ready channel that is closed once the engine has fully -// started. For proposal engine, this is true once the underlying consensus -// algorithm has started. -func (e *Engine) Ready() <-chan struct{} { - return e.unit.Ready(func() { - // Start up components for all epochs. This is typically a single epoch - // but can be multiple near epoch boundaries - epochs := make([]module.ReadyDoneAware, 0, len(e.epochs)) - for _, epoch := range e.epochs { - epochs = append(epochs, epoch) - epoch.Start(epoch.signalerCtx) // start every component using its own context - } - // wait for all engines to start - <-util.AllReady(epochs...) - }, func() { - // check the current phase on startup, in case we are in setup phase - // and haven't yet voted for the next root QC - finalSnapshot := e.state.Final() - phase, err := finalSnapshot.Phase() +// checkShouldVoteOnStartup checks whether we should vote, and if so, sends a signal +// to the worker thread responsible for voting. +// No errors are expected during normal operation. +func (e *Engine) checkShouldVoteOnStartup(finalSnapshot protocol.Snapshot) error { + // check the current phase on startup, in case we are in setup phase + // and haven't yet voted for the next root QC + phase, err := finalSnapshot.Phase() + if err != nil { + return fmt.Errorf("could not get epoch phase for finalized snapshot: %w", err) + } + if phase == flow.EpochPhaseSetup { + header, err := finalSnapshot.Head() if err != nil { - e.log.Fatal().Err(err).Msg("could not check phase") - return - } - if phase == flow.EpochPhaseSetup { - e.unit.Launch(func() { - e.onEpochSetupPhaseStarted(finalSnapshot.Epochs().Next()) - }) + return fmt.Errorf("could not get header for finalized snapshot: %w", err) } - }) + e.epochSetupPhaseStartedEvents <- header + } + return nil +} + +// Ready returns a ready channel that is closed once the engine has fully started. +// This is true when the engine-scoped worker threads have started, and all presently +// running epoch components (max 2) have started. +func (e *Engine) Ready() <-chan struct{} { + e.mu.RLock() + components := make([]module.ReadyDoneAware, 0, len(e.epochs)+1) + components = append(components, e.cm) + for _, epoch := range e.epochs { + components = append(components, epoch) + } + e.mu.RUnlock() + + return util.AllReady(components...) } // Done returns a done channel that is closed once the engine has fully stopped. +// This is true when the engine-scoped worker threads have stopped, and all presently +// running epoch components (max 2) have stopped. func (e *Engine) Done() <-chan struct{} { - return e.unit.Done(func() { - // Stop components for all epochs. This is typically a single epoch - // but can be multiple near epoch boundaries - e.unit.Lock() - epochs := make([]module.ReadyDoneAware, 0, len(e.epochs)) - for _, epoch := range e.epochs { - epochs = append(epochs, epoch) - } - e.unit.Unlock() - e.stopComponents() // stop all components using parent context - <-util.AllDone(epochs...) - }) + e.mu.RLock() + components := make([]module.ReadyDoneAware, 0, len(e.epochs)+1) + components = append(components, e.cm) + for _, epoch := range e.epochs { + components = append(components, epoch) + } + e.mu.RUnlock() + + return util.AllDone(components...) } // createEpochComponents instantiates and returns epoch-scoped components for // the given epoch, using the configured factory. -// -// Returns ErrNotAuthorizedForEpoch if this node is not authorized in the epoch. +// Error returns: +// - ErrNotAuthorizedForEpoch if this node is not authorized in the epoch. func (e *Engine) createEpochComponents(epoch protocol.Epoch) (*EpochComponents, error) { - - state, prop, sync, hot, aggregator, err := e.factory.Create(epoch) + state, prop, sync, hot, voteAggregator, timeoutAggregator, messageHub, err := e.factory.Create(epoch) if err != nil { return nil, fmt.Errorf("could not setup requirements for epoch (%d): %w", epoch, err) } - components := NewEpochComponents(state, prop, sync, hot, aggregator) - return components, err + components := NewEpochComponents(state, prop, sync, hot, voteAggregator, timeoutAggregator, messageHub) + return components, nil } // EpochTransition handles the epoch transition protocol event. +// NOTE: epochmgr.Engine will not restart trailing cluster consensus instances from previous epoch, +// therefore no need to handle dropped protocol events here (see issue below). +// TODO gracefully handle restarts in first 600 blocks of epoch https://github.com/dapperlabs/flow-go/issues/5659 func (e *Engine) EpochTransition(_ uint64, first *flow.Header) { - e.unit.Launch(func() { - err := e.onEpochTransition(first) - if err != nil { - // failing to complete epoch transition is a fatal error - e.log.Fatal().Err(err).Msg("failed to complete epoch transition") - } - }) + e.epochTransitionEvents <- first } // EpochSetupPhaseStarted handles the epoch setup phase started protocol event. +// NOTE: Ready will check if we start up in the EpochSetup phase at initialization and trigger QC voting. +// This handles dropped protocol events and restarts interrupting QC voting. func (e *Engine) EpochSetupPhaseStarted(_ uint64, first *flow.Header) { - e.unit.Launch(func() { - nextEpoch := e.state.AtBlockID(first.ID()).Epochs().Next() - e.onEpochSetupPhaseStarted(nextEpoch) - }) + e.epochSetupPhaseStartedEvents <- first +} + +// handleEpochEvents handles events relating to the epoch lifecycle: +// - EpochTransition protocol event - we start epoch components for the starting epoch, +// and schedule shutdown for the ending epoch +// - EpochSetupPhaseStarted protocol event - we submit our node's vote for our cluster's +// root block in the next epoch +// - epochStopEvents - signalled when a previously scheduled shutdown height is reached. +// We shut down components associated with the epoch. +func (e *Engine) handleEpochEvents(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + for { + select { + case <-ctx.Done(): + return + case firstBlock := <-e.epochTransitionEvents: + err := e.onEpochTransition(ctx, firstBlock) + if err != nil { + ctx.Throw(err) + } + case firstBlock := <-e.epochSetupPhaseStartedEvents: + nextEpoch := e.state.AtBlockID(firstBlock.ID()).Epochs().Next() + e.onEpochSetupPhaseStarted(ctx, nextEpoch) + case epochCounter := <-e.epochStopEvents: + err := e.stopEpochComponents(epochCounter) + if err != nil { + ctx.Throw(err) + } + } + } +} + +// handleEpochErrors checks for irrecoverable errors thrown from any components from +// some epoch, and handles them. Currently, handling them means simply throwing them +// to the engine-level signaller context, which should cause the node to crash. +// In the future, we could restart the failed epoch's components instead. +// Must be run as a goroutine. +func (e *Engine) handleEpochErrors(ctx irrecoverable.SignalerContext, errCh <-chan error) { + select { + case <-ctx.Done(): + return + case err := <-errCh: + if err != nil { + ctx.Throw(err) + } + } } // onEpochTransition is called when we transition to a new epoch. It arranges // to shut down the last epoch's components and starts up the new epoch's. -func (e *Engine) onEpochTransition(first *flow.Header) error { - e.unit.Lock() - defer e.unit.Unlock() - +// +// No errors are expected during normal operation. +func (e *Engine) onEpochTransition(ctx irrecoverable.SignalerContext, first *flow.Header) error { epoch := e.state.AtBlockID(first.ID()).Epochs().Current() counter, err := epoch.Counter() if err != nil { @@ -275,36 +286,36 @@ func (e *Engine) onEpochTransition(first *flow.Header) error { Logger() // exit early and log if the epoch already exists - _, exists := e.epochs[counter] + _, exists := e.getEpochComponents(counter) if exists { - log.Warn().Msg("epoch transition: components for new epoch already setup") + log.Warn().Msg("epoch transition: components for new epoch already setup, exiting...") return nil } + // register a callback to stop the just-ended epoch at the appropriate block height + e.prepareToStopEpochComponents(counter-1, lastEpochMaxHeight) + log.Info().Msg("epoch transition: creating components for new epoch...") // create components for new epoch components, err := e.createEpochComponents(epoch) - // if we are not authorized in this epoch, skip starting up cluster consensus - if errors.Is(err, ErrNotAuthorizedForEpoch) { - e.prepareToStopEpochComponents(counter-1, lastEpochMaxHeight) - return nil - } if err != nil { + if errors.Is(err, ErrNotAuthorizedForEpoch) { + // if we are not authorized in this epoch, skip starting up cluster consensus + log.Info().Msg("epoch transition: we are not authorized for new epoch, exiting...") + return nil + } return fmt.Errorf("could not create epoch components: %w", err) } // start up components - err = e.startEpochComponents(counter, components) + err = e.startEpochComponents(ctx, counter, components) if err != nil { - return fmt.Errorf("could not start epoch components: %w", err) + return fmt.Errorf("unexpected failure starting epoch components: %w", err) } log.Info().Msg("epoch transition: new epoch components started successfully") - // set up callback to stop previous epoch - e.prepareToStopEpochComponents(counter-1, lastEpochMaxHeight) - return nil } @@ -321,33 +332,16 @@ func (e *Engine) onEpochTransition(first *flow.Header) error { // can NOT be included by clusters in the new epoch, we MUST continue producing // these collections within the previous epoch's clusters. func (e *Engine) prepareToStopEpochComponents(epochCounter, epochMaxHeight uint64) { - stopAtHeight := epochMaxHeight + flow.DefaultTransactionExpiry + 1 - - log := e.log.With(). + e.log.Info(). Uint64("stopping_epoch_max_height", epochMaxHeight). Uint64("stopping_epoch_counter", epochCounter). Uint64("stop_at_height", stopAtHeight). Str("step", "epoch_transition"). - Logger() - - log.Info().Msgf("preparing to stop epoch components at height %d", stopAtHeight) + Msgf("preparing to stop epoch components at height %d", stopAtHeight) e.heightEvents.OnHeight(stopAtHeight, func() { - e.unit.Launch(func() { - e.unit.Lock() - defer e.unit.Unlock() - - log.Info().Msg("stopping components for previous epoch...") - - err := e.stopEpochComponents(epochCounter) - if err != nil { - e.log.Error().Err(err).Msgf("failed to stop components for epoch %d", epochCounter) - return - } - - log.Info().Msg("previous epoch components stopped successfully") - }) + e.epochStopEvents <- epochCounter }) } @@ -355,31 +349,30 @@ func (e *Engine) prepareToStopEpochComponents(epochCounter, epochMaxHeight uint6 // setup phase, or when the node is restarted during the epoch setup phase. It // kicks off setup tasks for the phase, in particular submitting a vote for the // next epoch's root cluster QC. -func (e *Engine) onEpochSetupPhaseStarted(nextEpoch protocol.Epoch) { - - ctx, cancel := context.WithCancel(e.unit.Ctx()) - defer cancel() +func (e *Engine) onEpochSetupPhaseStarted(ctx irrecoverable.SignalerContext, nextEpoch protocol.Epoch) { err := e.voter.Vote(ctx, nextEpoch) if err != nil { - e.log.Error().Err(err).Msg("failed to submit QC vote for next epoch") + if epochs.IsClusterQCNoVoteError(err) { + e.log.Warn().Err(err).Msg("unable to submit QC vote for next epoch") + return + } + ctx.Throw(fmt.Errorf("unexpected failure to submit QC vote for next epoch: %w", err)) } } // startEpochComponents starts the components for the given epoch and adds them // to the engine's internal mapping. -// -// CAUTION: the caller MUST acquire the engine lock. -func (e *Engine) startEpochComponents(counter uint64, components *EpochComponents) error { - - ctx, cancel := context.WithCancel(e.irrecoverableCtx) - signalerCtx, _ := irrecoverable.WithSignaler(ctx) +// No errors are expected during normal operation. +func (e *Engine) startEpochComponents(engineCtx irrecoverable.SignalerContext, counter uint64, components *EpochComponents) error { + epochCtx, cancel, errCh := irrecoverable.WithSignallerAndCancel(engineCtx) // start component using its own context - components.Start(signalerCtx) + components.Start(epochCtx) + go e.handleEpochErrors(engineCtx, errCh) select { case <-components.Ready(): - e.epochs[counter] = NewStartableEpochComponents(components, signalerCtx, cancel) + e.storeEpochComponents(counter, NewRunningEpochComponents(components, cancel)) return nil case <-time.After(e.startupTimeout): cancel() // cancel current context if we didn't start in time @@ -388,14 +381,14 @@ func (e *Engine) startEpochComponents(counter uint64, components *EpochComponent } // stopEpochComponents stops the components for the given epoch and removes them -// from the engine's internal mapping. -// -// CAUTION: the caller MUST acquire the engine lock. +// from the engine's internal mapping. If no components exit for the given epoch, +// this is a no-op and a warning is logged. +// No errors are expected during normal operation. func (e *Engine) stopEpochComponents(counter uint64) error { - - components, exists := e.epochs[counter] + components, exists := e.getEpochComponents(counter) if !exists { - return fmt.Errorf("can not stop non-existent epoch %d", counter) + e.log.Warn().Msgf("attempted to stop non-existent epoch %d", counter) + return nil } // stop individual component @@ -403,10 +396,36 @@ func (e *Engine) stopEpochComponents(counter uint64) error { select { case <-components.Done(): - delete(e.epochs, counter) + e.removeEpoch(counter) e.pools.ForEpoch(counter).Clear() return nil case <-time.After(e.startupTimeout): return fmt.Errorf("could not stop epoch %d components after %s", counter, e.startupTimeout) } } + +// getEpochComponents retrieves the stored (running) epoch components for the given epoch counter. +// If no epoch with the counter is stored, returns (nil, false). +// Safe for concurrent use. +func (e *Engine) getEpochComponents(counter uint64) (*RunningEpochComponents, bool) { + e.mu.RLock() + epoch, ok := e.epochs[counter] + e.mu.RUnlock() + return epoch, ok +} + +// storeEpochComponents stores the given epoch components in the engine's mapping. +// Safe for concurrent use. +func (e *Engine) storeEpochComponents(counter uint64, components *RunningEpochComponents) { + e.mu.Lock() + e.epochs[counter] = components + e.mu.Unlock() +} + +// removeEpoch removes the epoch components with the given counter. +// Safe for concurrent use. +func (e *Engine) removeEpoch(counter uint64) { + e.mu.Lock() + delete(e.epochs, counter) + e.mu.Unlock() +} diff --git a/engine/collection/epochmgr/engine_test.go b/engine/collection/epochmgr/engine_test.go index ab3b366d91f..b08c4c4c8a7 100644 --- a/engine/collection/epochmgr/engine_test.go +++ b/engine/collection/epochmgr/engine_test.go @@ -1,7 +1,7 @@ package epochmgr import ( - "io" + "context" "testing" "time" @@ -15,13 +15,14 @@ import ( epochmgr "github.com/onflow/flow-go/engine/collection/epochmgr/mock" "github.com/onflow/flow-go/model/flow" realmodule "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + mockcomponent "github.com/onflow/flow-go/module/component/mock" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/mempool/epochs" "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/module/metrics" - module "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/mocknetwork" + mockmodule "github.com/onflow/flow-go/module/mock" realcluster "github.com/onflow/flow-go/state/cluster" cluster "github.com/onflow/flow-go/state/cluster/mock" realprotocol "github.com/onflow/flow-go/state/protocol" @@ -33,30 +34,38 @@ import ( // mockComponents is a container for the mocked version of epoch components. type mockComponents struct { - state *cluster.State - prop *mocknetwork.Engine - sync *mocknetwork.Engine - hotstuff *module.HotStuff - aggregator *mockhotstuff.VoteAggregator + state *cluster.State + prop *mockcomponent.Component + sync *mockmodule.ReadyDoneAware + hotstuff *mockmodule.HotStuff + voteAggregator *mockhotstuff.VoteAggregator + timeoutAggregator *mockhotstuff.TimeoutAggregator + messageHub *mockcomponent.Component } func newMockComponents() *mockComponents { components := &mockComponents{ - state: new(cluster.State), - prop: new(mocknetwork.Engine), - sync: new(mocknetwork.Engine), - hotstuff: new(module.HotStuff), - aggregator: new(mockhotstuff.VoteAggregator), + state: new(cluster.State), + prop: new(mockcomponent.Component), + sync: new(mockmodule.ReadyDoneAware), + hotstuff: new(mockmodule.HotStuff), + voteAggregator: new(mockhotstuff.VoteAggregator), + timeoutAggregator: new(mockhotstuff.TimeoutAggregator), + messageHub: new(mockcomponent.Component), } unittest.ReadyDoneify(components.prop) unittest.ReadyDoneify(components.sync) unittest.ReadyDoneify(components.hotstuff) - unittest.ReadyDoneify(components.aggregator) + unittest.ReadyDoneify(components.voteAggregator) + unittest.ReadyDoneify(components.timeoutAggregator) + unittest.ReadyDoneify(components.messageHub) - // for now only aggregator and hotstuff supports module.Startable, mock only it + components.prop.On("Start", mock.Anything) components.hotstuff.On("Start", mock.Anything) - components.aggregator.On("Start", mock.Anything) + components.voteAggregator.On("Start", mock.Anything) + components.timeoutAggregator.On("Start", mock.Anything) + components.messageHub.On("Start", mock.Anything) return components } @@ -66,30 +75,36 @@ type Suite struct { // engine dependencies log zerolog.Logger - me *module.Local + me *mockmodule.Local state *protocol.State snap *protocol.Snapshot pools *epochs.TransactionPools // qc voter dependencies signer *mockhotstuff.Signer - client *module.QCContractClient - voter *module.ClusterRootQCVoter + client *mockmodule.QCContractClient + voter *mockmodule.ClusterRootQCVoter factory *epochmgr.EpochComponentsFactory heights *events.Heights epochQuery *mocks.EpochQuery counter uint64 // reflects the counter of the current epoch + phase flow.EpochPhase // phase at mocked snapshot + header *flow.Header // header at mocked snapshot epochs map[uint64]*protocol.Epoch // track all epochs components map[uint64]*mockComponents // track all epoch components + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error + engine *Engine } func (suite *Suite) SetupTest() { - suite.log = zerolog.New(io.Discard) - suite.me = new(module.Local) + suite.log = unittest.Logger() + suite.me = new(mockmodule.Local) suite.state = new(protocol.State) suite.snap = new(protocol.Snapshot) @@ -97,8 +112,8 @@ func (suite *Suite) SetupTest() { suite.components = make(map[uint64]*mockComponents) suite.signer = new(mockhotstuff.Signer) - suite.client = new(module.QCContractClient) - suite.voter = new(module.ClusterRootQCVoter) + suite.client = new(mockmodule.QCContractClient) + suite.voter = new(mockmodule.ClusterRootQCVoter) suite.factory = new(epochmgr.EpochComponentsFactory) suite.heights = new(events.Heights) @@ -113,18 +128,31 @@ func (suite *Suite) SetupTest() { }). Return( func(epoch realprotocol.Epoch) realcluster.State { return suite.ComponentsForEpoch(epoch).state }, - func(epoch realprotocol.Epoch) network.Engine { return suite.ComponentsForEpoch(epoch).prop }, - func(epoch realprotocol.Epoch) network.Engine { return suite.ComponentsForEpoch(epoch).sync }, + func(epoch realprotocol.Epoch) component.Component { return suite.ComponentsForEpoch(epoch).prop }, + func(epoch realprotocol.Epoch) realmodule.ReadyDoneAware { return suite.ComponentsForEpoch(epoch).sync }, func(epoch realprotocol.Epoch) realmodule.HotStuff { return suite.ComponentsForEpoch(epoch).hotstuff }, func(epoch realprotocol.Epoch) hotstuff.VoteAggregator { - return suite.ComponentsForEpoch(epoch).aggregator + return suite.ComponentsForEpoch(epoch).voteAggregator + }, + func(epoch realprotocol.Epoch) hotstuff.TimeoutAggregator { + return suite.ComponentsForEpoch(epoch).timeoutAggregator }, + func(epoch realprotocol.Epoch) component.Component { return suite.ComponentsForEpoch(epoch).messageHub }, func(epoch realprotocol.Epoch) error { return nil }, ) + suite.phase = flow.EpochPhaseSetup + suite.header = unittest.BlockHeaderFixture() suite.epochQuery = mocks.NewEpochQuery(suite.T(), suite.counter) suite.state.On("Final").Return(suite.snap) + suite.state.On("AtBlockID", suite.header.ID()).Return(suite.snap) suite.snap.On("Epochs").Return(suite.epochQuery) + suite.snap.On("Head").Return( + func() *flow.Header { return suite.header }, + func() error { return nil }) + suite.snap.On("Phase").Return( + func() flow.EpochPhase { return suite.phase }, + func() error { return nil }) // add current and next epochs suite.AddEpoch(suite.counter) @@ -139,6 +167,28 @@ func (suite *Suite) SetupTest() { suite.Require().Nil(err) } +// StartEngine starts the engine under test, and spawns a routine to check for irrecoverable errors. +func (suite *Suite) StartEngine() { + suite.ctx, suite.cancel, suite.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + go unittest.FailOnIrrecoverableError(suite.T(), suite.ctx.Done(), suite.errs) + suite.engine.Start(suite.ctx) + unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) +} + +// TearDownTest stops the engine and checks for any irrecoverable errors. +func (suite *Suite) TearDownTest() { + if suite.cancel == nil { + return + } + suite.cancel() + unittest.RequireCloseBefore(suite.T(), suite.engine.Done(), time.Second, "engine failed to stop") + select { + case err := <-suite.errs: + assert.NoError(suite.T(), err) + default: + } +} + func TestEpochManager(t *testing.T) { suite.Run(t, new(Suite)) } @@ -164,8 +214,8 @@ func (suite *Suite) AssertEpochStarted(counter uint64) { suite.Assert().True(ok, "asserting nonexistent epoch started", counter) components.prop.AssertCalled(suite.T(), "Ready") components.sync.AssertCalled(suite.T(), "Ready") - components.aggregator.AssertCalled(suite.T(), "Ready") - components.aggregator.AssertCalled(suite.T(), "Start", mock.Anything) + components.voteAggregator.AssertCalled(suite.T(), "Ready") + components.voteAggregator.AssertCalled(suite.T(), "Start", mock.Anything) } // AssertEpochStopped asserts that the components for the given epoch have been stopped. @@ -191,17 +241,18 @@ func (suite *Suite) MockAsUnauthorizedNode() { suite.factory = new(epochmgr.EpochComponentsFactory) suite.factory. On("Create", mock.Anything). - Return(nil, nil, nil, nil, nil, ErrNotAuthorizedForEpoch) + Return(nil, nil, nil, nil, nil, nil, nil, ErrNotAuthorizedForEpoch) var err error suite.engine, err = New(suite.log, suite.me, suite.state, suite.pools, suite.voter, suite.factory, suite.heights) suite.Require().Nil(err) } -// if we start up during the setup phase, we should kick off the root QC voter +// TestRestartInSetupPhase tests that, if we start up during the setup phase, +// we should kick off the root QC voter func (suite *Suite) TestRestartInSetupPhase() { - - suite.snap.On("Phase").Return(flow.EpochPhaseSetup, nil) + // we are in setup phase + suite.phase = flow.EpochPhaseSetup // should call voter with next epoch var called = make(chan struct{}) suite.voter.On("Vote", mock.Anything, suite.epochQuery.Next()). @@ -211,24 +262,22 @@ func (suite *Suite) TestRestartInSetupPhase() { }).Once() // start up the engine - unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) - unittest.AssertClosesBefore(suite.T(), called, time.Second) + suite.StartEngine() - suite.voter.AssertExpectations(suite.T()) + unittest.AssertClosesBefore(suite.T(), called, time.Second) + suite.voter.AssertExpectations(suite.T()) // TODO replace with new constructor } -// When a collection node joins the network at an epoch boundary, they must -// start running during the EpochSetup phase in the epoch before they become -// an authorized member so they submit their cluster QC vote. +// TestStartAsUnauthorizedNode test that when a collection node joins the network +// at an epoch boundary, they must start running during the EpochSetup phase in the +// epoch before they become an authorized member so they submit their cluster QC vote. // -// These nodes must kick off the root QC voter but should not attempt to -// participate in cluster consensus in the current epoch. +// These nodes must kick off the root QC voter but should not attempt to participate +// in cluster consensus in the current epoch. func (suite *Suite) TestStartAsUnauthorizedNode() { suite.MockAsUnauthorizedNode() - // we are in setup phase - suite.snap.On("Phase").Return(flow.EpochPhaseSetup, nil) - + suite.phase = flow.EpochPhaseSetup // should call voter with next epoch var called = make(chan struct{}) suite.voter.On("Vote", mock.Anything, suite.epochQuery.Next()). @@ -238,7 +287,7 @@ func (suite *Suite) TestStartAsUnauthorizedNode() { }).Once() // start the engine - unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + suite.StartEngine() // should have submitted vote unittest.AssertClosesBefore(suite.T(), called, time.Second) @@ -247,9 +296,11 @@ func (suite *Suite) TestStartAsUnauthorizedNode() { assert.Empty(suite.T(), suite.engine.epochs, "should have 0 epoch components") } -// should kick off root QC voter on setup phase start event +// TestRespondToPhaseChange should kick off root QC voter when we receive an event +// indicating the EpochSetup phase has started. func (suite *Suite) TestRespondToPhaseChange() { - + // start in staking phase + suite.phase = flow.EpochPhaseStaking // should call voter with next epoch var called = make(chan struct{}) suite.voter.On("Vote", mock.Anything, suite.epochQuery.Next()). @@ -258,46 +309,56 @@ func (suite *Suite) TestRespondToPhaseChange() { close(called) }).Once() - first := unittest.BlockHeaderFixture() - suite.state.On("AtBlockID", first.ID()).Return(suite.snap) + firstBlockOfEpochSetupPhase := unittest.BlockHeaderFixture() + suite.state.On("AtBlockID", firstBlockOfEpochSetupPhase.ID()).Return(suite.snap) + suite.StartEngine() - suite.engine.EpochSetupPhaseStarted(0, first) + // after receiving the protocol event, we should submit our root QC vote + suite.engine.EpochSetupPhaseStarted(0, firstBlockOfEpochSetupPhase) unittest.AssertClosesBefore(suite.T(), called, time.Second) suite.voter.AssertExpectations(suite.T()) } +// TestRespondToEpochTransition tests the engine's behaviour during epoch transition. +// It should: +// - instantiate cluster consensus for the new epoch +// - register callback to stop the previous epoch's cluster consensus +// - stop the previous epoch's cluster consensus when the callback is invoked func (suite *Suite) TestRespondToEpochTransition() { // we are in committed phase - suite.snap.On("Phase").Return(flow.EpochPhaseCommitted, nil) - - // start the engine - unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + suite.phase = flow.EpochPhaseCommitted + suite.StartEngine() - first := unittest.BlockHeaderFixture() - suite.state.On("AtBlockID", first.ID()).Return(suite.snap) + firstBlockOfEpoch := unittest.BlockHeaderFixture() + suite.state.On("AtBlockID", firstBlockOfEpoch.ID()).Return(suite.snap) // should set up callback for height at which previous epoch expires var expiryCallback func() - var done = make(chan struct{}) - suite.heights.On("OnHeight", first.Height+flow.DefaultTransactionExpiry, mock.Anything). + heightRegistered := make(chan struct{}) + suite.heights.On("OnHeight", firstBlockOfEpoch.Height+flow.DefaultTransactionExpiry, mock.Anything). Run(func(args mock.Arguments) { expiryCallback = args.Get(1).(func()) - close(done) + close(heightRegistered) }). Once() // mock the epoch transition suite.TransitionEpoch() // notify the engine of the epoch transition - suite.engine.EpochTransition(suite.counter, first) - unittest.AssertClosesBefore(suite.T(), done, time.Second) + suite.engine.EpochTransition(suite.counter, firstBlockOfEpoch) + // ensure we registered a height callback + unittest.AssertClosesBefore(suite.T(), heightRegistered, time.Second) suite.Assert().NotNil(expiryCallback) // the engine should have two epochs under management, the just ended epoch // and the newly started epoch - suite.Assert().Len(suite.engine.epochs, 2) + suite.Eventually(func() bool { + suite.engine.mu.Lock() + defer suite.engine.mu.Unlock() + return len(suite.engine.epochs) == 2 + }, time.Second, 10*time.Millisecond) _, exists := suite.engine.epochs[suite.counter-1] suite.Assert().True(exists, "should have previous epoch components") _, exists = suite.engine.epochs[suite.counter] @@ -311,10 +372,10 @@ func (suite *Suite) TestRespondToEpochTransition() { expiryCallback() suite.Assert().Eventually(func() bool { - suite.engine.unit.Lock() - defer suite.engine.unit.Unlock() + suite.engine.mu.Lock() + defer suite.engine.mu.Unlock() return len(suite.engine.epochs) == 1 - }, time.Second, time.Millisecond) + }, time.Second, 10*time.Millisecond) // after the previous epoch expires, we should only have current epoch _, exists = suite.engine.epochs[suite.counter] diff --git a/engine/collection/epochmgr/epoch_components.go b/engine/collection/epochmgr/epoch_components.go new file mode 100644 index 00000000000..426531e8bb9 --- /dev/null +++ b/engine/collection/epochmgr/epoch_components.go @@ -0,0 +1,102 @@ +package epochmgr + +import ( + "context" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/util" + "github.com/onflow/flow-go/state/cluster" +) + +// EpochComponents represents all dependencies for running an epoch. +type EpochComponents struct { + *component.ComponentManager + state cluster.State + comp component.Component + messageHub component.Component + sync module.ReadyDoneAware + hotstuff module.HotStuff + voteAggregator hotstuff.VoteAggregator + timeoutAggregator hotstuff.TimeoutAggregator +} + +var _ component.Component = (*EpochComponents)(nil) + +func NewEpochComponents( + state cluster.State, + comp component.Component, + sync module.ReadyDoneAware, + hotstuff module.HotStuff, + voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + messageHub component.Component, +) *EpochComponents { + components := &EpochComponents{ + state: state, + comp: comp, + sync: sync, + hotstuff: hotstuff, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, + messageHub: messageHub, + } + + builder := component.NewComponentManagerBuilder() + // start new worker that will start child components and wait for them to finish + builder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + // start components + voteAggregator.Start(ctx) + timeoutAggregator.Start(ctx) + hotstuff.Start(ctx) + comp.Start(ctx) + messageHub.Start(ctx) + // wait until all components start + <-util.AllReady( + components.hotstuff, + components.comp, + components.sync, + components.voteAggregator, + components.timeoutAggregator, + components.messageHub, + ) + + // signal that startup has finished, and we are ready to go + ready() + + // wait for shutdown to be commenced + <-ctx.Done() + // wait for compliance engine and event loop to shut down + <-util.AllDone( + components.hotstuff, + components.comp, + components.sync, + components.voteAggregator, + components.timeoutAggregator, + components.messageHub, + ) + }) + components.ComponentManager = builder.Build() + + return components +} + +// RunningEpochComponents contains all consensus-related components for an epoch +// and the cancel function to stop these components. All components must have been +// started when the RunningEpochComponents is constructed. +type RunningEpochComponents struct { + *EpochComponents + cancel context.CancelFunc // used to stop the epoch components +} + +// NewRunningEpochComponents returns a new RunningEpochComponents container for the +// given epoch components. The components must have already been started using some +// context, which the provided cancel function cancels (stopping the epoch components). +func NewRunningEpochComponents(components *EpochComponents, cancel context.CancelFunc) *RunningEpochComponents { + return &RunningEpochComponents{ + EpochComponents: components, + cancel: cancel, + } +} diff --git a/engine/collection/epochmgr/factories/proposal.go b/engine/collection/epochmgr/factories/compliance.go similarity index 77% rename from engine/collection/epochmgr/factories/proposal.go rename to engine/collection/epochmgr/factories/compliance.go index 2dcb46f5b31..777a5db03b6 100644 --- a/engine/collection/epochmgr/factories/proposal.go +++ b/engine/collection/epochmgr/factories/compliance.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/flow-go/engine/collection/compliance" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/buffer" + "github.com/onflow/flow-go/module/chainsync" modulecompliance "github.com/onflow/flow-go/module/compliance" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/state/cluster" @@ -16,7 +17,7 @@ import ( "github.com/onflow/flow-go/storage" ) -type ProposalEngineFactory struct { +type ComplianceEngineFactory struct { log zerolog.Logger me module.Local net network.Network @@ -28,8 +29,8 @@ type ProposalEngineFactory struct { complianceOpts []modulecompliance.Opt } -// NewFactory returns a new collection proposal engine factory. -func NewProposalEngineFactory( +// NewComplianceEngineFactory returns a new collection compliance engine factory. +func NewComplianceEngineFactory( log zerolog.Logger, net network.Network, me module.Local, @@ -39,9 +40,9 @@ func NewProposalEngineFactory( protoState protocol.State, transactions storage.Transactions, complianceOpts ...modulecompliance.Opt, -) (*ProposalEngineFactory, error) { +) (*ComplianceEngineFactory, error) { - factory := &ProposalEngineFactory{ + factory := &ComplianceEngineFactory{ log: log, me: me, net: net, @@ -55,11 +56,16 @@ func NewProposalEngineFactory( return factory, nil } -func (f *ProposalEngineFactory) Create( +func (f *ComplianceEngineFactory) Create( + hotstuffMetrics module.HotstuffMetrics, clusterState cluster.MutableState, headers storage.Headers, payloads storage.ClusterPayloads, + syncCore *chainsync.Core, + hot module.HotStuff, voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + validator hotstuff.Validator, ) (*compliance.Engine, error) { cache := buffer.NewPendingClusterBlocks() @@ -67,11 +73,16 @@ func (f *ProposalEngineFactory) Create( f.log, f.engMetrics, f.mempoolMetrics, + hotstuffMetrics, f.colMetrics, headers, clusterState, cache, + syncCore, + validator, + hot, voteAggregator, + timeoutAggregator, f.complianceOpts..., ) if err != nil { @@ -80,7 +91,6 @@ func (f *ProposalEngineFactory) Create( engine, err := compliance.NewEngine( f.log, - f.net, f.me, f.protoState, payloads, diff --git a/engine/collection/epochmgr/factories/epoch.go b/engine/collection/epochmgr/factories/epoch.go index d90671e8788..ca5bb9b03e4 100644 --- a/engine/collection/epochmgr/factories/epoch.go +++ b/engine/collection/epochmgr/factories/epoch.go @@ -6,9 +6,8 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/engine/collection/epochmgr" "github.com/onflow/flow-go/module" - chainsync "github.com/onflow/flow-go/module/chainsync" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/mempool/epochs" - "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/cluster/badger" "github.com/onflow/flow-go/state/protocol" @@ -16,33 +15,41 @@ import ( ) type EpochComponentsFactory struct { - me module.Local - pools *epochs.TransactionPools - builder *BuilderFactory - state *ClusterStateFactory - hotstuff *HotStuffFactory - proposal *ProposalEngineFactory - sync *SyncEngineFactory + me module.Local + pools *epochs.TransactionPools + builder *BuilderFactory + state *ClusterStateFactory + hotstuff *HotStuffFactory + compliance *ComplianceEngineFactory + sync *SyncEngineFactory + syncCore *SyncCoreFactory + messageHub *MessageHubFactory } +var _ epochmgr.EpochComponentsFactory = (*EpochComponentsFactory)(nil) + func NewEpochComponentsFactory( me module.Local, pools *epochs.TransactionPools, builder *BuilderFactory, state *ClusterStateFactory, hotstuff *HotStuffFactory, - proposal *ProposalEngineFactory, + compliance *ComplianceEngineFactory, + syncCore *SyncCoreFactory, sync *SyncEngineFactory, + messageHub *MessageHubFactory, ) *EpochComponentsFactory { factory := &EpochComponentsFactory{ - me: me, - pools: pools, - builder: builder, - state: state, - hotstuff: hotstuff, - proposal: proposal, - sync: sync, + me: me, + pools: pools, + builder: builder, + state: state, + hotstuff: hotstuff, + compliance: compliance, + syncCore: syncCore, + sync: sync, + messageHub: messageHub, } return factory } @@ -51,10 +58,12 @@ func (factory *EpochComponentsFactory) Create( epoch protocol.Epoch, ) ( state cluster.State, - proposal network.Engine, - sync network.Engine, + compliance component.Component, + sync module.ReadyDoneAware, hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + messageHub component.Component, err error, ) { @@ -100,7 +109,7 @@ func (factory *EpochComponentsFactory) Create( blocks storage.ClusterBlocks ) - stateRoot, err := badger.NewStateRoot(cluster.RootBlock()) + stateRoot, err := badger.NewStateRoot(cluster.RootBlock(), cluster.RootQC()) if err != nil { err = fmt.Errorf("could not create valid state root: %w", err) return @@ -127,39 +136,59 @@ func (factory *EpochComponentsFactory) Create( err = fmt.Errorf("could not create consensus modules: %w", err) return } - voteAggregator = hotstuffModules.Aggregator + voteAggregator = hotstuffModules.VoteAggregator + timeoutAggregator = hotstuffModules.TimeoutAggregator + validator := hotstuffModules.Validator - proposalEng, err := factory.proposal.Create(mutableState, headers, payloads, hotstuffModules.Aggregator) + hotstuff, err = factory.hotstuff.Create( + cluster, + state, + metrics, + builder, + headers, + hotstuffModules, + ) if err != nil { - err = fmt.Errorf("could not create proposal engine: %w", err) + err = fmt.Errorf("could not create hotstuff: %w", err) return } - var syncCore *chainsync.Core - syncCore, sync, err = factory.sync.Create(cluster.Members(), state, blocks, proposalEng) + syncCore, err := factory.syncCore.Create(cluster.ChainID()) if err != nil { - err = fmt.Errorf("could not create sync engine: %w", err) + err = fmt.Errorf("could not create sync core: %w", err) return } - hotstuff, err = factory.hotstuff.Create( - state, + + complianceEng, err := factory.compliance.Create( metrics, - builder, + mutableState, headers, - proposalEng, - hotstuffModules, + payloads, + syncCore, + hotstuff, + hotstuffModules.VoteAggregator, + hotstuffModules.TimeoutAggregator, + validator, ) if err != nil { - err = fmt.Errorf("could not create hotstuff: %w", err) + err = fmt.Errorf("could not create compliance engine: %w", err) return } + compliance = complianceEng + hotstuffModules.FinalizationDistributor.AddOnBlockFinalizedConsumer(complianceEng.OnFinalizedBlock) - hotstuffModules.FinalizationDistributor.AddOnBlockFinalizedConsumer(proposalEng.OnFinalizedBlock) + sync, err = factory.sync.Create(cluster.Members(), state, blocks, syncCore, complianceEng) + if err != nil { + err = fmt.Errorf("could not create sync engine: %w", err) + return + } - // attach dependencies to the proposal engine - proposal = proposalEng. - WithConsensus(hotstuff). - WithSync(syncCore) + clusterMessageHub, err := factory.messageHub.Create(state, payloads, hotstuff, complianceEng, hotstuffModules) + if err != nil { + err = fmt.Errorf("could not create message hub: %w", err) + } + hotstuffModules.Notifier.AddConsumer(clusterMessageHub) + messageHub = clusterMessageHub return } diff --git a/engine/collection/epochmgr/factories/hotstuff.go b/engine/collection/epochmgr/factories/hotstuff.go index 725ee4b46bd..d100cd65df7 100644 --- a/engine/collection/epochmgr/factories/hotstuff.go +++ b/engine/collection/epochmgr/factories/hotstuff.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/consensus/hotstuff/persister" + "github.com/onflow/flow-go/consensus/hotstuff/timeoutcollector" validatorImpl "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" "github.com/onflow/flow-go/consensus/hotstuff/votecollector" @@ -20,6 +21,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" hotmetrics "github.com/onflow/flow-go/module/metrics/hotstuff" + msig "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" @@ -28,12 +30,14 @@ import ( type HotStuffMetricsFunc func(chainID flow.ChainID) module.HotstuffMetrics type HotStuffFactory struct { - log zerolog.Logger - me module.Local - db *badger.DB - protoState protocol.State - createMetrics HotStuffMetricsFunc - opts []consensus.Option + baseLogger zerolog.Logger + me module.Local + db *badger.DB + protoState protocol.State + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + createMetrics HotStuffMetricsFunc + opts []consensus.Option } func NewHotStuffFactory( @@ -41,17 +45,21 @@ func NewHotStuffFactory( me module.Local, db *badger.DB, protoState protocol.State, + engineMetrics module.EngineMetrics, + mempoolMetrics module.MempoolMetrics, createMetrics HotStuffMetricsFunc, opts ...consensus.Option, ) (*HotStuffFactory, error) { factory := &HotStuffFactory{ - log: log, - me: me, - db: db, - protoState: protoState, - createMetrics: createMetrics, - opts: opts, + baseLogger: log, + me: me, + db: db, + protoState: protoState, + engineMetrics: engineMetrics, + mempoolMetrics: mempoolMetrics, + createMetrics: createMetrics, + opts: opts, } return factory, nil } @@ -64,19 +72,19 @@ func (f *HotStuffFactory) CreateModules( payloads storage.ClusterPayloads, updater module.Finalizer, ) (*consensus.HotstuffModules, module.HotstuffMetrics, error) { - // setup metrics/logging with the new chain ID + log := f.createLogger(cluster) metrics := f.createMetrics(cluster.ChainID()) notifier := pubsub.NewDistributor() finalizationDistributor := pubsub.NewFinalizationDistributor() notifier.AddConsumer(finalizationDistributor) - notifier.AddConsumer(notifications.NewLogConsumer(f.log)) + notifier.AddConsumer(notifications.NewLogConsumer(log)) notifier.AddConsumer(hotmetrics.NewMetricsConsumer(metrics)) - notifier.AddConsumer(notifications.NewTelemetryConsumer(f.log, cluster.ChainID())) + notifier.AddConsumer(notifications.NewTelemetryConsumer(log)) var ( err error - committee hotstuff.Committee + committee hotstuff.DynamicCommittee ) committee, err = committees.NewClusterCommittee(f.protoState, payloads, cluster, epoch, f.me.NodeID()) if err != nil { @@ -108,10 +116,13 @@ func (f *HotStuffFactory) CreateModules( qcDistributor := pubsub.NewQCCreatedDistributor() verifier := verification.NewStakingVerifier() - validator := validatorImpl.NewMetricsWrapper(validatorImpl.New(committee, forks, verifier), metrics) + validator := validatorImpl.NewMetricsWrapper(validatorImpl.New(committee, verifier), metrics) voteProcessorFactory := votecollector.NewStakingVoteProcessorFactory(committee, qcDistributor.OnQcConstructedFromVotes) - aggregator, err := consensus.NewVoteAggregator( - f.log, + voteAggregator, err := consensus.NewVoteAggregator( + log, + metrics, + f.engineMetrics, + f.mempoolMetrics, // since we don't want to aggregate votes for finalized view, // the lowest retained view starts with the next view of the last finalized view. finalizedBlock.View+1, @@ -123,25 +134,44 @@ func (f *HotStuffFactory) CreateModules( return nil, nil, err } + timeoutCollectorDistributor := pubsub.NewTimeoutCollectorDistributor() + timeoutProcessorFactory := timeoutcollector.NewTimeoutProcessorFactory(log, timeoutCollectorDistributor, committee, validator, msig.CollectorTimeoutTag) + + timeoutAggregator, err := consensus.NewTimeoutAggregator( + log, + metrics, + f.engineMetrics, + f.mempoolMetrics, + notifier, + timeoutProcessorFactory, + timeoutCollectorDistributor, + finalizedBlock.View+1, + ) + if err != nil { + return nil, nil, err + } + return &consensus.HotstuffModules{ - Forks: forks, - Validator: validator, - Notifier: notifier, - Committee: committee, - Signer: signer, - Persist: persister.New(f.db, cluster.ChainID()), - Aggregator: aggregator, - QCCreatedDistributor: qcDistributor, - FinalizationDistributor: finalizationDistributor, + Forks: forks, + Validator: validator, + Notifier: notifier, + Committee: committee, + Signer: signer, + Persist: persister.New(f.db, cluster.ChainID()), + VoteAggregator: voteAggregator, + TimeoutAggregator: timeoutAggregator, + QCCreatedDistributor: qcDistributor, + TimeoutCollectorDistributor: timeoutCollectorDistributor, + FinalizationDistributor: finalizationDistributor, }, metrics, nil } func (f *HotStuffFactory) Create( + cluster protocol.Cluster, clusterState cluster.State, metrics module.HotstuffMetrics, builder module.Builder, headers storage.Headers, - communicator hotstuff.Communicator, hotstuffModules *consensus.HotstuffModules, ) (module.HotStuff, error) { @@ -153,11 +183,11 @@ func (f *HotStuffFactory) Create( return nil, err } + log := f.createLogger(cluster) participant, err := consensus.NewParticipant( - f.log, + log, metrics, builder, - communicator, finalizedBlock, pendingBlocks, hotstuffModules, @@ -165,3 +195,8 @@ func (f *HotStuffFactory) Create( ) return participant, err } + +// createLogger creates a logger by wrapping base logger by decorating it will cluster ID +func (f *HotStuffFactory) createLogger(cluster protocol.Cluster) zerolog.Logger { + return f.baseLogger.With().Str("chain", cluster.ChainID().String()).Logger() +} diff --git a/engine/collection/epochmgr/factories/hub.go b/engine/collection/epochmgr/factories/hub.go new file mode 100644 index 00000000000..434a699aef7 --- /dev/null +++ b/engine/collection/epochmgr/factories/hub.go @@ -0,0 +1,58 @@ +package factories + +import ( + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus" + "github.com/onflow/flow-go/engine/collection" + "github.com/onflow/flow-go/engine/collection/message_hub" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +type MessageHubFactory struct { + log zerolog.Logger + me module.Local + net network.Network + protoState protocol.State + engineMetrics module.EngineMetrics +} + +func NewMessageHubFactory(log zerolog.Logger, + net network.Network, + me module.Local, + engineMetrics module.EngineMetrics, + protoState protocol.State) *MessageHubFactory { + return &MessageHubFactory{ + log: log, + me: me, + net: net, + protoState: protoState, + engineMetrics: engineMetrics, + } +} + +func (f *MessageHubFactory) Create( + clusterState cluster.State, + payloads storage.ClusterPayloads, + hotstuff module.HotStuff, + compliance collection.Compliance, + hotstuffModules *consensus.HotstuffModules, +) (*message_hub.MessageHub, error) { + return message_hub.NewMessageHub( + f.log, + f.engineMetrics, + f.net, + f.me, + compliance, + hotstuff, + hotstuffModules.VoteAggregator, + hotstuffModules.TimeoutAggregator, + f.protoState, + clusterState, + payloads, + ) +} diff --git a/engine/collection/epochmgr/factories/sync.go b/engine/collection/epochmgr/factories/sync.go index 19cc2fcb153..98bf10c4142 100644 --- a/engine/collection/epochmgr/factories/sync.go +++ b/engine/collection/epochmgr/factories/sync.go @@ -3,11 +3,11 @@ package factories import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/collection" syncengine "github.com/onflow/flow-go/engine/collection/synchronization" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - chainsync "github.com/onflow/flow-go/module/chainsync" - "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/chainsync" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/storage" @@ -18,7 +18,6 @@ type SyncEngineFactory struct { net network.Network me module.Local metrics module.EngineMetrics - conf chainsync.Config } func NewSyncEngineFactory( @@ -26,7 +25,6 @@ func NewSyncEngineFactory( metrics module.EngineMetrics, net network.Network, me module.Local, - conf chainsync.Config, ) (*SyncEngineFactory, error) { factory := &SyncEngineFactory{ @@ -34,7 +32,6 @@ func NewSyncEngineFactory( me: me, net: net, metrics: metrics, - conf: conf, } return factory, nil } @@ -43,13 +40,10 @@ func (f *SyncEngineFactory) Create( participants flow.IdentityList, state cluster.State, blocks storage.ClusterBlocks, - comp network.Engine, -) (*chainsync.Core, *syncengine.Engine, error) { + core *chainsync.Core, + comp collection.Compliance, +) (*syncengine.Engine, error) { - core, err := chainsync.New(f.log, f.conf, metrics.NewChainSyncCollector()) - if err != nil { - return nil, nil, err - } engine, err := syncengine.New( f.log, f.metrics, @@ -61,5 +55,5 @@ func (f *SyncEngineFactory) Create( comp, core, ) - return core, engine, err + return engine, err } diff --git a/engine/collection/epochmgr/factories/sync_core.go b/engine/collection/epochmgr/factories/sync_core.go new file mode 100644 index 00000000000..55d1ca0fa66 --- /dev/null +++ b/engine/collection/epochmgr/factories/sync_core.go @@ -0,0 +1,37 @@ +package factories + +import ( + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/chainsync" + "github.com/onflow/flow-go/module/metrics" +) + +type SyncCoreFactory struct { + log zerolog.Logger + conf chainsync.Config +} + +func NewSyncCoreFactory( + log zerolog.Logger, + conf chainsync.Config, +) (*SyncCoreFactory, error) { + factory := &SyncCoreFactory{ + log: log, + conf: conf, + } + return factory, nil +} + +func (f *SyncCoreFactory) Create(chainID flow.ChainID) (*chainsync.Core, error) { + core, err := chainsync.New( + f.log.With().Str("sync_chain_id", chainID.String()).Logger(), + f.conf, + metrics.NewChainSyncCollector(chainID), + chainID) + if err != nil { + return nil, err + } + return core, nil +} diff --git a/engine/collection/epochmgr/factory.go b/engine/collection/epochmgr/factory.go index 589431da802..801f314ff87 100644 --- a/engine/collection/epochmgr/factory.go +++ b/engine/collection/epochmgr/factory.go @@ -3,7 +3,7 @@ package epochmgr import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/state/protocol" ) @@ -20,10 +20,12 @@ type EpochComponentsFactory interface { // Must return ErrNotAuthorizedForEpoch if this node is not authorized in the epoch. Create(epoch protocol.Epoch) ( state cluster.State, - proposal network.Engine, - sync network.Engine, + proposal component.Component, + sync module.ReadyDoneAware, hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + messageHub component.Component, err error, ) } diff --git a/engine/collection/epochmgr/mock/epoch_components_factory.go b/engine/collection/epochmgr/mock/epoch_components_factory.go index 320d62c9543..aebaaf0ec57 100644 --- a/engine/collection/epochmgr/mock/epoch_components_factory.go +++ b/engine/collection/epochmgr/mock/epoch_components_factory.go @@ -3,6 +3,7 @@ package mock import ( + component "github.com/onflow/flow-go/module/component" cluster "github.com/onflow/flow-go/state/cluster" hotstuff "github.com/onflow/flow-go/consensus/hotstuff" @@ -11,8 +12,6 @@ import ( module "github.com/onflow/flow-go/module" - network "github.com/onflow/flow-go/network" - protocol "github.com/onflow/flow-go/state/protocol" ) @@ -22,7 +21,7 @@ type EpochComponentsFactory struct { } // Create provides a mock function with given fields: epoch -func (_m *EpochComponentsFactory) Create(epoch protocol.Epoch) (cluster.State, network.Engine, network.Engine, module.HotStuff, hotstuff.VoteAggregator, error) { +func (_m *EpochComponentsFactory) Create(epoch protocol.Epoch) (cluster.State, component.Component, module.ReadyDoneAware, module.HotStuff, hotstuff.VoteAggregator, hotstuff.TimeoutAggregator, component.Component, error) { ret := _m.Called(epoch) var r0 cluster.State @@ -34,21 +33,21 @@ func (_m *EpochComponentsFactory) Create(epoch protocol.Epoch) (cluster.State, n } } - var r1 network.Engine - if rf, ok := ret.Get(1).(func(protocol.Epoch) network.Engine); ok { + var r1 component.Component + if rf, ok := ret.Get(1).(func(protocol.Epoch) component.Component); ok { r1 = rf(epoch) } else { if ret.Get(1) != nil { - r1 = ret.Get(1).(network.Engine) + r1 = ret.Get(1).(component.Component) } } - var r2 network.Engine - if rf, ok := ret.Get(2).(func(protocol.Epoch) network.Engine); ok { + var r2 module.ReadyDoneAware + if rf, ok := ret.Get(2).(func(protocol.Epoch) module.ReadyDoneAware); ok { r2 = rf(epoch) } else { if ret.Get(2) != nil { - r2 = ret.Get(2).(network.Engine) + r2 = ret.Get(2).(module.ReadyDoneAware) } } @@ -70,14 +69,32 @@ func (_m *EpochComponentsFactory) Create(epoch protocol.Epoch) (cluster.State, n } } - var r5 error - if rf, ok := ret.Get(5).(func(protocol.Epoch) error); ok { + var r5 hotstuff.TimeoutAggregator + if rf, ok := ret.Get(5).(func(protocol.Epoch) hotstuff.TimeoutAggregator); ok { r5 = rf(epoch) } else { - r5 = ret.Error(5) + if ret.Get(5) != nil { + r5 = ret.Get(5).(hotstuff.TimeoutAggregator) + } + } + + var r6 component.Component + if rf, ok := ret.Get(6).(func(protocol.Epoch) component.Component); ok { + r6 = rf(epoch) + } else { + if ret.Get(6) != nil { + r6 = ret.Get(6).(component.Component) + } + } + + var r7 error + if rf, ok := ret.Get(7).(func(protocol.Epoch) error); ok { + r7 = rf(epoch) + } else { + r7 = ret.Error(7) } - return r0, r1, r2, r3, r4, r5 + return r0, r1, r2, r3, r4, r5, r6, r7 } type mockConstructorTestingTNewEpochComponentsFactory interface { diff --git a/engine/collection/message_hub/message_hub.go b/engine/collection/message_hub/message_hub.go new file mode 100644 index 00000000000..9a34f357293 --- /dev/null +++ b/engine/collection/message_hub/message_hub.go @@ -0,0 +1,477 @@ +package message_hub + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/collection" + "github.com/onflow/flow-go/engine/common/fifoqueue" + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + clusterkv "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/logging" +) + +// defaultMessageHubRequestsWorkers number of workers to dispatch events for requests +const defaultMessageHubRequestsWorkers = 5 + +// defaultProposalQueueCapacity number of pending outgoing proposals stored in queue +const defaultProposalQueueCapacity = 3 + +// defaultVoteQueueCapacity number of pending outgoing votes stored in queue +const defaultVoteQueueCapacity = 20 + +// defaultTimeoutQueueCapacity number of pending outgoing timeouts stored in queue +const defaultTimeoutQueueCapacity = 3 + +// packedVote is a helper structure to pack recipientID and vote into one structure to pass through fifoqueue.FifoQueue +type packedVote struct { + recipientID flow.Identifier + vote *messages.ClusterBlockVote +} + +// MessageHub is a central module for handling incoming and outgoing messages via cluster consensus channel. +// It performs message routing for incoming messages by matching them by type and sending to respective engine. +// For incoming messages handling processing looks like this: +// +// +-------------------+ +------------+ +// -->| Cluster-Channel |----->| MessageHub | +// +-------------------+ +------+-----+ +// ------------|------------ +// +------+---------+ | +------+-----+ | +------+------------+ +// | VoteAggregator |----+ | Compliance | +----| TimeoutAggregator | +// +----------------+ +------------+ +------+------------+ +// vote block timeout object +// +// MessageHub acts as communicator and handles hotstuff.Consumer communication events to send votes, broadcast timeouts +// and proposals. It is responsible for communication between cluster consensus participants. +// It implements hotstuff.Consumer interface and needs to be subscribed for notifications via pub/sub. +// All communicator events are handled on worker thread to prevent sender from blocking. +// For outgoing messages processing logic looks like this: +// +// +-------------------+ +------------+ +----------+ +------------------------+ +// | Cluster-Channel |<-----| MessageHub |<-----| Consumer |<-----| Hotstuff | +// +-------------------+ +------+-----+ +----------+ +------------------------+ +// pub/sub vote, timeout, proposal +// +// MessageHub is safe to use in concurrent environment. +type MessageHub struct { + *component.ComponentManager + notifications.NoopConsumer + log zerolog.Logger + me module.Local + engineMetrics module.EngineMetrics + state protocol.State + payloads storage.ClusterPayloads + con network.Conduit + ownOutboundMessageNotifier engine.Notifier + ownOutboundVotes *fifoqueue.FifoQueue // queue for handling outgoing vote transmissions + ownOutboundProposals *fifoqueue.FifoQueue // queue for handling outgoing proposal transmissions + ownOutboundTimeouts *fifoqueue.FifoQueue // queue for handling outgoing timeout transmissions + clusterIdentityFilter flow.IdentityFilter + + // injected dependencies + compliance collection.Compliance // handler of incoming block proposals + hotstuff module.HotStuff // used to submit proposals that were previously broadcast + voteAggregator hotstuff.VoteAggregator // handler of incoming votes + timeoutAggregator hotstuff.TimeoutAggregator // handler of incoming timeouts +} + +var _ network.MessageProcessor = (*MessageHub)(nil) +var _ hotstuff.CommunicatorConsumer = (*MessageHub)(nil) + +// NewMessageHub constructs new instance of message hub +// No errors are expected during normal operations. +func NewMessageHub(log zerolog.Logger, + engineMetrics module.EngineMetrics, + net network.Network, + me module.Local, + compliance collection.Compliance, + hotstuff module.HotStuff, + voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + state protocol.State, + clusterState clusterkv.State, + payloads storage.ClusterPayloads, +) (*MessageHub, error) { + // find my cluster for the current epoch + // TODO this should flow from cluster state as source of truth + clusters, err := state.Final().Epochs().Current().Clustering() + if err != nil { + return nil, fmt.Errorf("could not get clusters: %w", err) + } + currentCluster, _, found := clusters.ByNodeID(me.NodeID()) + if !found { + return nil, fmt.Errorf("could not find cluster for self") + } + + ownOutboundVotes, err := fifoqueue.NewFifoQueue(defaultVoteQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize votes queue") + } + ownOutboundProposals, err := fifoqueue.NewFifoQueue(defaultProposalQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize blocks queue") + } + ownOutboundTimeouts, err := fifoqueue.NewFifoQueue(defaultTimeoutQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize timeouts queue") + } + hub := &MessageHub{ + log: log.With().Str("engine", "cluster_message_hub").Logger(), + me: me, + engineMetrics: engineMetrics, + state: state, + payloads: payloads, + compliance: compliance, + hotstuff: hotstuff, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, + ownOutboundMessageNotifier: engine.NewNotifier(), + ownOutboundVotes: ownOutboundVotes, + ownOutboundProposals: ownOutboundProposals, + ownOutboundTimeouts: ownOutboundTimeouts, + clusterIdentityFilter: filter.And( + filter.In(currentCluster), + filter.Not(filter.HasNodeID(me.NodeID())), + ), + } + + // register network conduit + chainID, err := clusterState.Params().ChainID() + if err != nil { + return nil, fmt.Errorf("could not get chain ID: %w", err) + } + conduit, err := net.Register(channels.ConsensusCluster(chainID), hub) + if err != nil { + return nil, fmt.Errorf("could not register engine: %w", err) + } + hub.con = conduit + + componentBuilder := component.NewComponentManagerBuilder() + // This implementation tolerates if the networking layer sometimes blocks on send requests. + // We use by default 5 go-routines here. This is fine, because outbound messages are temporally sparse + // under normal operations. Hence, the go-routines should mostly be asleep waiting for work. + for i := 0; i < defaultMessageHubRequestsWorkers; i++ { + componentBuilder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + hub.queuedMessagesProcessingLoop(ctx) + }) + } + hub.ComponentManager = componentBuilder.Build() + return hub, nil +} + +// queuedMessagesProcessingLoop orchestrates dispatching of previously queued messages +func (h *MessageHub) queuedMessagesProcessingLoop(ctx irrecoverable.SignalerContext) { + notifier := h.ownOutboundMessageNotifier.Channel() + for { + select { + case <-ctx.Done(): + return + case <-notifier: + err := h.sendOwnMessages(ctx) + if err != nil { + ctx.Throw(fmt.Errorf("internal error processing queued messages: %w", err)) + return + } + } + } +} + +// sendOwnMessages is a function which dispatches previously queued messages on worker thread +// This function is called whenever we have queued messages ready to be dispatched. +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnMessages(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + default: + } + + msg, ok := h.ownOutboundProposals.Pop() + if ok { + block := msg.(*flow.Header) + err := h.sendOwnProposal(block) + if err != nil { + return fmt.Errorf("could not process queued block %v: %w", block.ID(), err) + } + continue + } + + msg, ok = h.ownOutboundVotes.Pop() + if ok { + packed := msg.(*packedVote) + err := h.sendOwnVote(packed) + if err != nil { + return fmt.Errorf("could not process queued vote: %w", err) + } + continue + } + + msg, ok = h.ownOutboundTimeouts.Pop() + if ok { + err := h.sendOwnTimeout(msg.(*model.TimeoutObject)) + if err != nil { + return fmt.Errorf("coult not process queued timeout: %w", err) + } + continue + } + + // when there is no more messages in the queue, back to the loop to wait + // for the next incoming message to arrive. + return nil + } +} + +// sendOwnTimeout propagates the timeout to the consensus committee (excluding myself) +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnTimeout(timeout *model.TimeoutObject) error { + log := timeout.LogContext(h.log).Logger() + log.Info().Msg("processing timeout broadcast request from hotstuff") + + // Retrieve all collection nodes in our cluster (excluding myself). + recipients, err := h.state.Final().Identities(h.clusterIdentityFilter) + if err != nil { + return fmt.Errorf("could not get cluster members for broadcasting timeout: %w", err) + } + // create the timeout message + msg := &messages.ClusterTimeoutObject{ + View: timeout.View, + NewestQC: timeout.NewestQC, + LastViewTC: timeout.LastViewTC, + SigData: timeout.SigData, + TimeoutTick: timeout.TimeoutTick, + } + + err = h.con.Publish(msg, recipients.NodeIDs()...) + if err != nil { + if !errors.Is(err, network.EmptyTargetList) { + log.Err(err).Msg("could not broadcast timeout") + } + return nil + } + log.Info().Msg("cluster timeout was broadcast") + h.engineMetrics.MessageSent(metrics.EngineCollectionMessageHub, metrics.MessageTimeoutObject) + + return nil +} + +// sendOwnVote propagates the vote via unicast to another node that is the next leader +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnVote(packed *packedVote) error { + log := h.log.With(). + Hex("collection_id", packed.vote.BlockID[:]). + Uint64("collection_view", packed.vote.View). + Hex("recipient_id", packed.recipientID[:]). + Logger() + log.Info().Msg("processing vote transmission request from hotstuff") + + // send the vote the desired recipient + err := h.con.Unicast(packed.vote, packed.recipientID) + if err != nil { + log.Err(err).Msg("could not send vote") + return nil + } + log.Info().Msg("collection vote transmitted") + h.engineMetrics.MessageSent(metrics.EngineCollectionMessageHub, metrics.MessageBlockVote) + + return nil +} + +// sendOwnProposal propagates the block proposal to the consensus committee by broadcasting to all other cluster participants (excluding myself) +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnProposal(header *flow.Header) error { + // first, check that we are the proposer of the block + if header.ProposerID != h.me.NodeID() { + return fmt.Errorf("cannot broadcast proposal with non-local proposer (%x)", header.ProposerID) + } + + // retrieve the payload for the block + payload, err := h.payloads.ByBlockID(header.ID()) + if err != nil { + return fmt.Errorf("could not retrieve payload for proposal: %w", err) + } + + log := h.log.With(). + Str("chain_id", header.ChainID.String()). + Uint64("block_height", header.Height). + Uint64("block_view", header.View). + Hex("block_id", logging.ID(header.ID())). + Hex("parent_id", header.ParentID[:]). + Hex("ref_block", payload.ReferenceBlockID[:]). + Int("transaction_count", payload.Collection.Len()). + Hex("parent_signer_indices", header.ParentVoterIndices). + Logger() + + log.Debug().Msg("processing cluster broadcast request from hotstuff") + + // retrieve all collection nodes in our cluster + recipients, err := h.state.Final().Identities(h.clusterIdentityFilter) + if err != nil { + return fmt.Errorf("could not get cluster members for broadcasting collection proposal") + } + + // create the proposal message for the collection + proposal := messages.NewClusterBlockProposal(&cluster.Block{ + Header: header, + Payload: payload, + }) + + // broadcast the proposal to consensus nodes + err = h.con.Publish(proposal, recipients.NodeIDs()...) + if err != nil { + if !errors.Is(err, network.EmptyTargetList) { + log.Err(err).Msg("could not send proposal message") + } + return nil + } + log.Info().Msg("cluster proposal was broadcast") + h.engineMetrics.MessageSent(metrics.EngineCollectionMessageHub, metrics.MessageBlockProposal) + + return nil +} + +// OnOwnVote propagates the vote to relevant recipient(s): +// - [common case] vote is queued and is sent via unicast to another node that is the next leader by worker +// - [special case] this node is the next leader: vote is directly forwarded to the node's internal `VoteAggregator` +func (h *MessageHub) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + vote := &messages.ClusterBlockVote{ + BlockID: blockID, + View: view, + SigData: sigData, + } + + // special case: I am the next leader + if recipientID == h.me.NodeID() { + h.forwardToOwnVoteAggregator(vote, h.me.NodeID()) // forward vote to my own `voteAggregator` + return + } + + // common case: someone else is leader + packed := &packedVote{ + recipientID: recipientID, + vote: vote, + } + if ok := h.ownOutboundVotes.Push(packed); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineCollectionMessageHub, metrics.MessageBlockVote) + } +} + +// OnOwnTimeout forwards timeout to node's internal `timeoutAggregator` and queues timeout for +// subsequent propagation to all consensus participants (excluding this node) +func (h *MessageHub) OnOwnTimeout(timeout *model.TimeoutObject) { + h.forwardToOwnTimeoutAggregator(timeout) // forward timeout to my own `timeoutAggregator` + if ok := h.ownOutboundTimeouts.Push(timeout); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineCollectionMessageHub, metrics.MessageTimeoutObject) + } +} + +// OnOwnProposal directly forwards proposal to HotStuff core logic(skipping compliance engine as we assume our +// own proposals to be correct) and queues proposal for subsequent propagation to all consensus participants (including this node). +// The proposal will only be placed in the queue, after the specified delay (or dropped on shutdown signal). +func (h *MessageHub) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + go func() { + select { + case <-time.After(time.Until(targetPublicationTime)): + case <-h.ShutdownSignal(): + return + } + + hotstuffProposal := model.ProposalFromFlow(proposal) + // notify vote aggregator that new block proposal is available, in case we are next leader + h.voteAggregator.AddBlock(hotstuffProposal) // non-blocking + + // TODO(active-pacemaker): replace with pub/sub? + // submit proposal to our own processing pipeline + h.hotstuff.SubmitProposal(hotstuffProposal) // non-blocking + + if ok := h.ownOutboundProposals.Push(proposal); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineCollectionMessageHub, metrics.MessageBlockProposal) + } + }() +} + +// Process handles incoming messages from consensus channel. After matching message by type, sends it to the correct +// component for handling. +// No errors are expected during normal operations. +func (h *MessageHub) Process(channel channels.Channel, originID flow.Identifier, message interface{}) error { + switch msg := message.(type) { + case *messages.ClusterBlockProposal: + h.compliance.OnClusterBlockProposal(flow.Slashable[messages.ClusterBlockProposal]{ + OriginID: originID, + Message: msg, + }) + case *messages.ClusterBlockVote: + h.forwardToOwnVoteAggregator(msg, originID) + case *messages.ClusterTimeoutObject: + t := &model.TimeoutObject{ + View: msg.View, + NewestQC: msg.NewestQC, + LastViewTC: msg.LastViewTC, + SignerID: originID, + SigData: msg.SigData, + TimeoutTick: msg.TimeoutTick, + } + h.forwardToOwnTimeoutAggregator(t) + default: + h.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, message, channel) + } + return nil +} + +// forwardToOwnVoteAggregator converts vote to generic `model.Vote`, logs vote and forwards it to own `voteAggregator`. +// Per API convention, timeoutAggregator` is non-blocking, hence, this call returns quickly. +func (h *MessageHub) forwardToOwnVoteAggregator(vote *messages.ClusterBlockVote, originID flow.Identifier) { + h.engineMetrics.MessageReceived(metrics.EngineCollectionMessageHub, metrics.MessageBlockVote) + v := &model.Vote{ + View: vote.View, + BlockID: vote.BlockID, + SignerID: originID, + SigData: vote.SigData, + } + h.log.Info(). + Uint64("block_view", v.View). + Hex("block_id", v.BlockID[:]). + Hex("voter", v.SignerID[:]). + Str("vote_id", v.ID().String()). + Msg("block vote received, forwarding block vote to hotstuff vote aggregator") + h.voteAggregator.AddVote(v) +} + +// forwardToOwnTimeoutAggregator logs timeout and forwards it to own `timeoutAggregator`. +// Per API convention, timeoutAggregator` is non-blocking, hence, this call returns quickly. +func (h *MessageHub) forwardToOwnTimeoutAggregator(t *model.TimeoutObject) { + h.engineMetrics.MessageReceived(metrics.EngineCollectionMessageHub, metrics.MessageTimeoutObject) + h.log.Info(). + Hex("origin_id", t.SignerID[:]). + Uint64("view", t.View). + Str("timeout_id", t.ID().String()). + Msg("timeout received, forwarding timeout to hotstuff timeout aggregator") + h.timeoutAggregator.AddTimeout(t) +} diff --git a/engine/collection/message_hub/message_hub_test.go b/engine/collection/message_hub/message_hub_test.go new file mode 100644 index 00000000000..16aa4e729a7 --- /dev/null +++ b/engine/collection/message_hub/message_hub_test.go @@ -0,0 +1,356 @@ +package message_hub + +import ( + "context" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + mockcollection "github.com/onflow/flow-go/engine/collection/mock" + "github.com/onflow/flow-go/model/cluster" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/module/util" + netint "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/mocknetwork" + clusterint "github.com/onflow/flow-go/state/cluster" + clusterstate "github.com/onflow/flow-go/state/cluster/mock" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storerr "github.com/onflow/flow-go/storage" + storage "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestMessageHub(t *testing.T) { + suite.Run(t, new(MessageHubSuite)) +} + +// MessageHubSuite tests the cluster consensus message hub. Holds mocked dependencies that are used by different test scenarios. +type MessageHubSuite struct { + suite.Suite + + // parameters + cluster flow.IdentityList + clusterID flow.ChainID + myID flow.Identifier + head *cluster.Block + + // mocked dependencies + payloads *storage.ClusterPayloads + me *module.Local + state *clusterstate.MutableState + protoState *protocol.MutableState + net *mocknetwork.Network + con *mocknetwork.Conduit + hotstuff *module.HotStuff + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator + compliance *mockcollection.Compliance + snapshot *clusterstate.Snapshot + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error + hub *MessageHub +} + +func (s *MessageHubSuite) SetupTest() { + // seed the RNG + rand.Seed(time.Now().UnixNano()) + + // initialize the paramaters + s.cluster = unittest.IdentityListFixture(3, + unittest.WithRole(flow.RoleCollection), + unittest.WithWeight(1000), + ) + s.myID = s.cluster[0].NodeID + s.clusterID = "cluster-id" + block := unittest.ClusterBlockFixture() + s.head = &block + + s.payloads = storage.NewClusterPayloads(s.T()) + s.me = module.NewLocal(s.T()) + s.protoState = protocol.NewMutableState(s.T()) + s.net = mocknetwork.NewNetwork(s.T()) + s.con = mocknetwork.NewConduit(s.T()) + s.hotstuff = module.NewHotStuff(s.T()) + s.voteAggregator = hotstuff.NewVoteAggregator(s.T()) + s.timeoutAggregator = hotstuff.NewTimeoutAggregator(s.T()) + s.compliance = mockcollection.NewCompliance(s.T()) + + // set up proto state mock + protoEpoch := &protocol.Epoch{} + clusters := flow.ClusterList{s.cluster} + protoEpoch.On("Clustering").Return(clusters, nil) + + protoQuery := &protocol.EpochQuery{} + protoQuery.On("Current").Return(protoEpoch) + + protoSnapshot := &protocol.Snapshot{} + protoSnapshot.On("Epochs").Return(protoQuery) + protoSnapshot.On("Identities", mock.Anything).Return( + func(selector flow.IdentityFilter) flow.IdentityList { + return s.cluster.Filter(selector) + }, + nil, + ) + s.protoState.On("Final").Return(protoSnapshot) + + // set up cluster state mock + s.state = &clusterstate.MutableState{} + s.state.On("Final").Return( + func() clusterint.Snapshot { + return s.snapshot + }, + ) + s.state.On("AtBlockID", mock.Anything).Return( + func(blockID flow.Identifier) clusterint.Snapshot { + return s.snapshot + }, + ) + clusterParams := &protocol.Params{} + clusterParams.On("ChainID").Return(s.clusterID, nil) + + s.state.On("Params").Return(clusterParams) + + // set up local module mock + s.me.On("NodeID").Return( + func() flow.Identifier { + return s.myID + }, + ).Maybe() + + // set up network module mock + s.net.On("Register", mock.Anything, mock.Anything).Return( + func(channel channels.Channel, engine netint.MessageProcessor) netint.Conduit { + return s.con + }, + nil, + ) + + // set up protocol snapshot mock + s.snapshot = &clusterstate.Snapshot{} + s.snapshot.On("Head").Return( + func() *flow.Header { + return s.head.Header + }, + nil, + ) + + engineMetrics := metrics.NewNoopCollector() + hub, err := NewMessageHub( + unittest.Logger(), + engineMetrics, + s.net, + s.me, + s.compliance, + s.hotstuff, + s.voteAggregator, + s.timeoutAggregator, + s.protoState, + s.state, + s.payloads, + ) + require.NoError(s.T(), err) + s.hub = hub + + s.ctx, s.cancel, s.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + s.hub.Start(s.ctx) + + unittest.AssertClosesBefore(s.T(), s.hub.Ready(), time.Second) +} + +// TearDownTest stops the hub and checks there are no errors thrown to the SignallerContext. +func (s *MessageHubSuite) TearDownTest() { + s.cancel() + unittest.RequireCloseBefore(s.T(), s.hub.Done(), time.Second, "hub failed to stop") + select { + case err := <-s.errs: + assert.NoError(s.T(), err) + default: + } +} + +// TestProcessIncomingMessages tests processing of incoming messages, MessageHub matches messages by type +// and sends them to other modules which execute business logic. +func (s *MessageHubSuite) TestProcessIncomingMessages() { + var channel channels.Channel + originID := unittest.IdentifierFixture() + s.Run("to-compliance-engine", func() { + block := unittest.ClusterBlockFixture() + + blockProposalMsg := messages.NewClusterBlockProposal(&block) + expectedComplianceMsg := flow.Slashable[messages.ClusterBlockProposal]{ + OriginID: originID, + Message: blockProposalMsg, + } + s.compliance.On("OnClusterBlockProposal", expectedComplianceMsg).Return(nil).Once() + err := s.hub.Process(channel, originID, blockProposalMsg) + require.NoError(s.T(), err) + }) + s.Run("to-vote-aggregator", func() { + expectedVote := unittest.VoteFixture(unittest.WithVoteSignerID(originID)) + msg := &messages.ClusterBlockVote{ + View: expectedVote.View, + BlockID: expectedVote.BlockID, + SigData: expectedVote.SigData, + } + s.voteAggregator.On("AddVote", expectedVote) + err := s.hub.Process(channel, originID, msg) + require.NoError(s.T(), err) + }) + s.Run("to-timeout-aggregator", func() { + expectedTimeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectSignerID(originID)) + msg := &messages.ClusterTimeoutObject{ + View: expectedTimeout.View, + NewestQC: expectedTimeout.NewestQC, + LastViewTC: expectedTimeout.LastViewTC, + SigData: expectedTimeout.SigData, + } + s.timeoutAggregator.On("AddTimeout", expectedTimeout) + err := s.hub.Process(channel, originID, msg) + require.NoError(s.T(), err) + }) + s.Run("unsupported-msg-type", func() { + err := s.hub.Process(channel, originID, struct{}{}) + require.NoError(s.T(), err) + }) +} + +// TestOnOwnProposal tests broadcasting proposals with different inputs +func (s *MessageHubSuite) TestOnOwnProposal() { + // add execution node to cluster to make sure we exclude them from broadcast + s.cluster = append(s.cluster, unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution))) + + // generate a parent with height and chain ID set + parent := unittest.ClusterBlockFixture() + parent.Header.ChainID = "test" + parent.Header.Height = 10 + + // create a block with the parent and store the payload with correct ID + block := unittest.ClusterBlockWithParent(&parent) + block.Header.ProposerID = s.myID + + s.payloads.On("ByBlockID", block.Header.ID()).Return(block.Payload, nil) + s.payloads.On("ByBlockID", mock.Anything).Return(nil, storerr.ErrNotFound) + + s.Run("should fail with wrong proposer", func() { + header := *block.Header + header.ProposerID = unittest.IdentifierFixture() + err := s.hub.sendOwnProposal(&header) + require.Error(s.T(), err, "should fail with wrong proposer") + header.ProposerID = s.myID + }) + + // should fail since we can't query payload + s.Run("should fail with changed/missing parent", func() { + header := *block.Header + header.ParentID[0]++ + err := s.hub.sendOwnProposal(&header) + require.Error(s.T(), err, "should fail with missing parent") + header.ParentID[0]-- + }) + + // should fail with wrong block ID (payload unavailable) + s.Run("should fail with wrong block ID", func() { + header := *block.Header + header.View++ + err := s.hub.sendOwnProposal(&header) + require.Error(s.T(), err, "should fail with missing payload") + header.View-- + }) + + s.Run("should broadcast proposal and pass to HotStuff for valid proposals", func() { + expectedBroadcastMsg := messages.NewClusterBlockProposal(&block) + + submitted := make(chan struct{}) // closed when proposal is submitted to hotstuff + hotstuffProposal := model.ProposalFromFlow(block.Header) + s.voteAggregator.On("AddBlock", hotstuffProposal).Once() + s.hotstuff.On("SubmitProposal", hotstuffProposal). + Run(func(args mock.Arguments) { close(submitted) }). + Once() + + broadcast := make(chan struct{}) // closed when proposal is broadcast + s.con.On("Publish", expectedBroadcastMsg, s.cluster[1].NodeID, s.cluster[2].NodeID). + Run(func(_ mock.Arguments) { close(broadcast) }). + Return(nil). + Once() + + // submit to broadcast proposal + s.hub.OnOwnProposal(block.Header, time.Now()) + + unittest.AssertClosesBefore(s.T(), util.AllClosed(broadcast, submitted), time.Second) + }) +} + +// TestProcessMultipleMessagesHappyPath tests submitting all types of messages through full processing pipeline and +// asserting that expected message transmissions happened as expected. +func (s *MessageHubSuite) TestProcessMultipleMessagesHappyPath() { + var wg sync.WaitGroup + + s.Run("vote", func() { + wg.Add(1) + // prepare vote fixture + vote := unittest.VoteFixture() + recipientID := unittest.IdentifierFixture() + s.con.On("Unicast", mock.Anything, recipientID).Run(func(_ mock.Arguments) { + wg.Done() + }).Return(nil) + + // submit vote + s.hub.OnOwnVote(vote.BlockID, vote.View, vote.SigData, recipientID) + }) + s.Run("timeout", func() { + wg.Add(1) + // prepare timeout fixture + timeout := helper.TimeoutObjectFixture() + expectedBroadcastMsg := &messages.ClusterTimeoutObject{ + View: timeout.View, + NewestQC: timeout.NewestQC, + LastViewTC: timeout.LastViewTC, + SigData: timeout.SigData, + } + s.con.On("Publish", expectedBroadcastMsg, s.cluster[1].NodeID, s.cluster[2].NodeID). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil) + s.timeoutAggregator.On("AddTimeout", timeout).Once() + // submit timeout + s.hub.OnOwnTimeout(timeout) + }) + s.Run("proposal", func() { + wg.Add(1) + // prepare proposal fixture + proposal := unittest.ClusterBlockWithParent(s.head) + proposal.Header.ProposerID = s.myID + s.payloads.On("ByBlockID", proposal.Header.ID()).Return(proposal.Payload, nil) + + // unset chain and height to make sure they are correctly reconstructed + hotstuffProposal := model.ProposalFromFlow(proposal.Header) + s.voteAggregator.On("AddBlock", hotstuffProposal) + s.hotstuff.On("SubmitProposal", hotstuffProposal) + expectedBroadcastMsg := messages.NewClusterBlockProposal(&proposal) + s.con.On("Publish", expectedBroadcastMsg, s.cluster[1].NodeID, s.cluster[2].NodeID). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil) + + // submit proposal + s.hub.OnOwnProposal(proposal.Header, time.Now()) + }) + + unittest.RequireReturnsBefore(s.T(), func() { + wg.Wait() + }, time.Second, "expect to process messages before timeout") +} diff --git a/engine/collection/mock/compliance.go b/engine/collection/mock/compliance.go new file mode 100644 index 00000000000..305836762d7 --- /dev/null +++ b/engine/collection/mock/compliance.go @@ -0,0 +1,79 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + + messages "github.com/onflow/flow-go/model/messages" + + mock "github.com/stretchr/testify/mock" +) + +// Compliance is an autogenerated mock type for the Compliance type +type Compliance struct { + mock.Mock +} + +// Done provides a mock function with given fields: +func (_m *Compliance) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// OnClusterBlockProposal provides a mock function with given fields: proposal +func (_m *Compliance) OnClusterBlockProposal(proposal flow.Slashable[messages.ClusterBlockProposal]) { + _m.Called(proposal) +} + +// OnSyncedClusterBlock provides a mock function with given fields: block +func (_m *Compliance) OnSyncedClusterBlock(block flow.Slashable[messages.ClusterBlockProposal]) { + _m.Called(block) +} + +// Ready provides a mock function with given fields: +func (_m *Compliance) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *Compliance) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewCompliance interface { + mock.TestingT + Cleanup(func()) +} + +// NewCompliance creates a new instance of Compliance. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCompliance(t mockConstructorTestingTNewCompliance) *Compliance { + mock := &Compliance{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/collection/rpc/engine.go b/engine/collection/rpc/engine.go index 6172896c292..d81de798d76 100644 --- a/engine/collection/rpc/engine.go +++ b/engine/collection/rpc/engine.go @@ -18,7 +18,6 @@ import ( "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/grpcutils" ) // Backend defines the core functionality required by the RPC API. @@ -31,7 +30,7 @@ type Backend interface { // Config defines the configurable options for the ingress server. type Config struct { ListenAddr string - MaxMsgSize int // in bytes + MaxMsgSize uint // in bytes RpcMetricsEnabled bool // enable GRPC metrics } @@ -54,14 +53,10 @@ func New( apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the gRPC API e.g. Ping->100, ExecuteScriptAtBlockID->300 apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the gRPC API e.g. Ping->50, ExecuteScriptAtBlockID->10 ) *Engine { - if config.MaxMsgSize == 0 { - config.MaxMsgSize = grpcutils.DefaultMaxMsgSize - } - // create a GRPC server to serve GRPC clients grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(config.MaxMsgSize), - grpc.MaxSendMsgSize(config.MaxMsgSize), + grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors diff --git a/engine/collection/synchronization/engine.go b/engine/collection/synchronization/engine.go index 185c6306cf5..02d75c392a6 100644 --- a/engine/collection/synchronization/engine.go +++ b/engine/collection/synchronization/engine.go @@ -3,6 +3,7 @@ package synchronization import ( + "errors" "fmt" "math/rand" "time" @@ -11,10 +12,10 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/collection" "github.com/onflow/flow-go/engine/common/fifoqueue" commonsync "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/model/chainsync" - "github.com/onflow/flow-go/model/events" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" @@ -43,7 +44,7 @@ type Engine struct { me module.Local participants flow.IdentityList con network.Conduit - comp network.Engine // compliance layer engine + comp collection.Compliance // compliance layer engine pollInterval time.Duration scanInterval time.Duration @@ -66,7 +67,7 @@ func New( participants flow.IdentityList, state cluster.State, blocks storage.ClusterBlocks, - comp network.Engine, + comp collection.Compliance, core module.SyncCore, opts ...commonsync.OptionFunc, ) (*Engine, error) { @@ -298,14 +299,18 @@ func (e *Engine) onSyncResponse(originID flow.Identifier, res *messages.SyncResp func (e *Engine) onBlockResponse(originID flow.Identifier, res *messages.ClusterBlockResponse) { // process the blocks one by one for _, block := range res.Blocks { - if !e.core.HandleBlock(&block.Header) { + header := block.Header + if !e.core.HandleBlock(&header) { continue } - synced := &events.SyncedClusterBlock{ + synced := flow.Slashable[messages.ClusterBlockProposal]{ OriginID: originID, - Block: block, + Message: &messages.ClusterBlockProposal{ + Block: block, + }, } - e.comp.SubmitLocal(synced) + // forward the block to the compliance engine for validation and processing + e.comp.OnSyncedClusterBlock(synced) } } @@ -362,7 +367,7 @@ func (e *Engine) pollHeight() { Height: head.Height, } err = e.con.Multicast(req, synccore.DefaultPollNodes, e.participants.NodeIDs()...) - if err != nil { + if err != nil && !errors.Is(err, network.EmptyTargetList) { e.log.Warn().Err(err).Msg("sending sync request to poll heights failed") return } diff --git a/engine/collection/synchronization/engine_test.go b/engine/collection/synchronization/engine_test.go index 06607bed473..06799cc6ddf 100644 --- a/engine/collection/synchronization/engine_test.go +++ b/engine/collection/synchronization/engine_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/engine" + mockcollection "github.com/onflow/flow-go/engine/collection/mock" clustermodel "github.com/onflow/flow-go/model/cluster" - "github.com/onflow/flow-go/model/events" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" @@ -51,7 +51,7 @@ type SyncSuite struct { snapshot *cluster.Snapshot params *cluster.Params blocks *storage.ClusterBlocks - comp *mocknetwork.Engine + comp *mockcollection.Compliance core *module.SyncCore e *Engine } @@ -153,8 +153,7 @@ func (ss *SyncSuite) SetupTest() { ) // set up compliance engine mock - ss.comp = &mocknetwork.Engine{} - ss.comp.On("SubmitLocal", mock.Anything).Return() + ss.comp = mockcollection.NewCompliance(ss.T()) // set up sync core ss.core = &module.SyncCore{} @@ -339,7 +338,7 @@ func (ss *SyncSuite) TestOnRangeRequest() { var err error config := chainsync.DefaultConfig() config.MaxSize = 2 - ss.e.requestHandler.core, err = chainsync.New(ss.e.log, config, metrics.NewNoopCollector()) + ss.e.requestHandler.core, err = chainsync.New(ss.e.log, config, metrics.NewNoopCollector(), flow.Localnet) require.NoError(ss.T(), err) err = ss.e.requestHandler.onRangeRequest(originID, req) @@ -417,7 +416,7 @@ func (ss *SyncSuite) TestOnBatchRequest() { var err error config := chainsync.DefaultConfig() config.MaxSize = 2 - ss.e.requestHandler.core, err = chainsync.New(ss.e.log, config, metrics.NewNoopCollector()) + ss.e.requestHandler.core, err = chainsync.New(ss.e.log, config, metrics.NewNoopCollector(), flow.Localnet) require.NoError(ss.T(), err) err = ss.e.requestHandler.onBatchRequest(originID, req) @@ -444,12 +443,13 @@ func (ss *SyncSuite) TestOnBlockResponse() { ss.core.On("HandleBlock", unprocessable.Header).Return(false) res.Blocks = append(res.Blocks, messages.UntrustedClusterBlockFromInternal(&unprocessable)) - ss.comp.On("SubmitLocal", mock.Anything).Run(func(args mock.Arguments) { - res := args.Get(0).(*events.SyncedBlock) - ss.Assert().Equal(&processable, res.Block) + ss.comp.On("OnSyncedClusterBlock", mock.Anything).Run(func(args mock.Arguments) { + res := args.Get(0).(flow.Slashable[messages.ClusterBlockProposal]) + converted := res.Message.Block.ToInternal() + ss.Assert().Equal(processable.Header, converted.Header) + ss.Assert().Equal(processable.Payload, converted.Payload) ss.Assert().Equal(originID, res.OriginID) - }, - ) + }).Return(nil) ss.e.onBlockResponse(originID, res) ss.comp.AssertExpectations(ss.T()) diff --git a/engine/collection/test/cluster_switchover_test.go b/engine/collection/test/cluster_switchover_test.go index dad7ff7e248..1ce0b0ce747 100644 --- a/engine/collection/test/cluster_switchover_test.go +++ b/engine/collection/test/cluster_switchover_test.go @@ -90,7 +90,7 @@ func NewClusterSwitchoverTestCase(t *testing.T, conf ClusterSwitchoverTestConf) // create a root snapshot with the given number of initial clusters root, result, seal := unittest.BootstrapFixture(tc.identities) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(root.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) @@ -215,6 +215,7 @@ func (tc *ClusterSwitchoverTestCase) StartNodes() { // start all node components nodes := make([]module.ReadyDoneAware, 0, len(tc.nodes)) for _, node := range tc.nodes { + node.Start(tc.T()) nodes = append(nodes, node) } diff --git a/engine/common/follower/engine.go b/engine/common/follower/engine.go index e7031c038ae..b261b4fcd24 100644 --- a/engine/common/follower/engine.go +++ b/engine/common/follower/engine.go @@ -8,12 +8,17 @@ import ( "github.com/hashicorp/go-multierror" "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/model/events" + "github.com/onflow/flow-go/engine/common/fifoqueue" + "github.com/onflow/flow-go/engine/consensus" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/network" @@ -24,23 +29,34 @@ import ( "github.com/onflow/flow-go/utils/logging" ) +// defaultBlockQueueCapacity maximum capacity of inbound queue for `messages.BlockProposal`s +const defaultBlockQueueCapacity = 10_000 + +// Engine follows and maintains the local copy of the protocol state. It is a +// passive (read-only) version of the compliance engine. The compliance engine +// is employed by consensus nodes (active consensus participants) where the +// Follower engine is employed by all other node roles. +// Implements consensus.Compliance interface. type Engine struct { - unit *engine.Unit - log zerolog.Logger - config compliance.Config - me module.Local - engMetrics module.EngineMetrics - mempoolMetrics module.MempoolMetrics - cleaner storage.Cleaner - headers storage.Headers - payloads storage.Payloads - state protocol.MutableState - pending module.PendingBlockBuffer - follower module.HotStuffFollower - con network.Conduit - sync module.BlockRequester - tracer module.Tracer - channel channels.Channel + *component.ComponentManager + log zerolog.Logger + config compliance.Config + me module.Local + engMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + cleaner storage.Cleaner + headers storage.Headers + payloads storage.Payloads + state protocol.MutableState + pending module.PendingBlockBuffer + follower module.HotStuffFollower + validator hotstuff.Validator + con network.Conduit + sync module.BlockRequester + tracer module.Tracer + channel channels.Channel + pendingBlocks *fifoqueue.FifoQueue // queues for processing inbound blocks + pendingBlocksNotifier engine.Notifier } type Option func(*Engine) @@ -61,6 +77,9 @@ func WithChannel(channel channels.Channel) Option { } } +var _ network.MessageProcessor = (*Engine)(nil) +var _ consensus.Compliance = (*Engine)(nil) + func New( log zerolog.Logger, net network.Network, @@ -73,26 +92,35 @@ func New( state protocol.MutableState, pending module.PendingBlockBuffer, follower module.HotStuffFollower, + validator hotstuff.Validator, sync module.BlockRequester, tracer module.Tracer, opts ...Option, ) (*Engine, error) { + // FIFO queue for block proposals + pendingBlocks, err := fifoqueue.NewFifoQueue(defaultBlockQueueCapacity) + if err != nil { + return nil, fmt.Errorf("failed to create queue for inbound blocks: %w", err) + } + e := &Engine{ - unit: engine.NewUnit(), - log: log.With().Str("engine", "follower").Logger(), - config: compliance.DefaultConfig(), - me: me, - engMetrics: engMetrics, - mempoolMetrics: mempoolMetrics, - cleaner: cleaner, - headers: headers, - payloads: payloads, - state: state, - pending: pending, - follower: follower, - sync: sync, - tracer: tracer, - channel: channels.ReceiveBlocks, + log: log.With().Str("engine", "follower").Logger(), + config: compliance.DefaultConfig(), + me: me, + engMetrics: engMetrics, + mempoolMetrics: mempoolMetrics, + cleaner: cleaner, + headers: headers, + payloads: payloads, + state: state, + pending: pending, + follower: follower, + validator: validator, + sync: sync, + tracer: tracer, + channel: channels.ReceiveBlocks, + pendingBlocks: pendingBlocks, + pendingBlocksNotifier: engine.NewNotifier(), } for _, apply := range opts { @@ -105,124 +133,110 @@ func New( } e.con = con - return e, nil -} - -// Ready returns a ready channel that is closed once the engine has fully -// started. For consensus engine, this is true once the underlying consensus -// algorithm has started. -func (e *Engine) Ready() <-chan struct{} { - return e.unit.Ready(func() { - <-e.follower.Ready() - }) -} + e.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(e.processBlocksLoop). + Build() -// Done returns a done channel that is closed once the engine has fully stopped. -// For the consensus engine, we wait for hotstuff to finish. -func (e *Engine) Done() <-chan struct{} { - return e.unit.Done(func() { - <-e.follower.Done() - }) + return e, nil } -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - e.unit.Launch(func() { - err := e.process(e.me.NodeID(), event) - if err != nil { - engine.LogError(e.log, err) - } - }) +// OnBlockProposal errors when called since follower engine doesn't support direct ingestion via internal method. +func (e *Engine) OnBlockProposal(_ flow.Slashable[messages.BlockProposal]) { + e.log.Error().Msg("received unexpected block proposal via internal method") } -// Submit submits the given event from the node with the given origin ID -// for processing in a non-blocking manner. It returns instantly and logs -// a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { - e.unit.Launch(func() { - err := e.Process(channel, originID, event) - if err != nil { - engine.LogError(e.log, err) - } - }) -} +// OnSyncedBlock performs processing of incoming block by pushing into queue and notifying worker. +func (e *Engine) OnSyncedBlock(synced flow.Slashable[messages.BlockProposal]) { + e.engMetrics.MessageReceived(metrics.EngineFollower, metrics.MessageSyncedBlock) + // a block that is synced has to come locally, from the synchronization engine + // the block itself will contain the proposer to indicate who created it -// ProcessLocal processes an event originating on the local node. -func (e *Engine) ProcessLocal(event interface{}) error { - return e.unit.Do(func() error { - return e.process(e.me.NodeID(), event) - }) + // queue proposal + if e.pendingBlocks.Push(synced) { + e.pendingBlocksNotifier.Notify() + } } // Process processes the given event from the node with the given origin ID in // a blocking manner. It returns the potential processing error when done. -func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - return e.unit.Do(func() error { - return e.process(originID, event) - }) -} - -func (e *Engine) process(originID flow.Identifier, input interface{}) error { - switch v := input.(type) { - case *messages.BlockResponse: - e.engMetrics.MessageReceived(metrics.EngineFollower, metrics.MessageBlockResponse) - defer e.engMetrics.MessageHandled(metrics.EngineFollower, metrics.MessageBlockResponse) - e.unit.Lock() - defer e.unit.Unlock() - return e.onBlockResponse(originID, v) - case *events.SyncedBlock: - e.engMetrics.MessageReceived(metrics.EngineFollower, metrics.MessageSyncedBlock) - defer e.engMetrics.MessageHandled(metrics.EngineFollower, metrics.MessageSyncedBlock) - e.unit.Lock() - defer e.unit.Unlock() - return e.onSyncedBlock(originID, v) +func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, message interface{}) error { + switch msg := message.(type) { case *messages.BlockProposal: - e.engMetrics.MessageReceived(metrics.EngineFollower, metrics.MessageBlockProposal) - defer e.engMetrics.MessageHandled(metrics.EngineFollower, metrics.MessageBlockProposal) - e.unit.Lock() - defer e.unit.Unlock() - return e.onBlockProposal(originID, v, false) + e.onBlockProposal(flow.Slashable[messages.BlockProposal]{ + OriginID: originID, + Message: msg, + }) default: - return fmt.Errorf("invalid event type (%T)", input) + e.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, message, channel) } + return nil } -func (e *Engine) onSyncedBlock(originID flow.Identifier, synced *events.SyncedBlock) error { - - // a block that is synced has to come locally, from the synchronization engine - // the block itself will contain the proposer to indicate who created it - if originID != e.me.NodeID() { - return fmt.Errorf("synced block with non-local origin (local: %x, origin: %x)", e.me.NodeID(), originID) - } - - // process as proposal - proposal := &messages.BlockProposal{ - Block: synced.Block, +// processBlocksLoop processes available block, vote, and timeout messages as they are queued. +func (e *Engine) processBlocksLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + doneSignal := ctx.Done() + newMessageSignal := e.pendingBlocksNotifier.Channel() + for { + select { + case <-doneSignal: + return + case <-newMessageSignal: + err := e.processQueuedBlocks(doneSignal) // no errors expected during normal operations + if err != nil { + ctx.Throw(err) + } + } } - return e.onBlockProposal(originID, proposal, false) } -func (e *Engine) onBlockResponse(originID flow.Identifier, res *messages.BlockResponse) error { - for i, block := range res.Blocks { - proposal := &messages.BlockProposal{ - Block: block, +// processQueuedBlocks processes any available messages until the message queue is empty. +// Only returns when all inbound queues are empty (or the engine is terminated). +// No errors are expected during normal operation. All returned exceptions are potential +// symptoms of internal state corruption and should be fatal. +func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { + for { + select { + case <-doneSignal: + return nil + default: } - // process block proposal with a wait - if err := e.onBlockProposal(originID, proposal, true); err != nil { - return fmt.Errorf("fail to process the block at index %v in a range block response that contains %v blocks: %w", i, len(res.Blocks), err) + msg, ok := e.pendingBlocks.Pop() + if ok { + in := msg.(flow.Slashable[messages.BlockProposal]) + err := e.processBlockProposal(in.OriginID, in.Message) + if err != nil { + return fmt.Errorf("could not handle block proposal: %w", err) + } + e.engMetrics.MessageHandled(metrics.EngineFollower, metrics.MessageBlockProposal) + continue } + + // when there are no more messages in the queue, back to the processBlocksLoop to wait + // for the next incoming message to arrive. + return nil } +} - return nil +// onBlockProposal performs processing of incoming block by pushing into queue and notifying worker. +func (e *Engine) onBlockProposal(proposal flow.Slashable[messages.BlockProposal]) { + e.engMetrics.MessageReceived(metrics.EngineFollower, metrics.MessageBlockProposal) + // queue proposal + if e.pendingBlocks.Push(proposal) { + e.pendingBlocksNotifier.Notify() + } } -// onBlockProposal handles incoming block proposals. inRangeBlockResponse will determine whether or not we should wait in processBlockAndDescendants -func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal, inRangeBlockResponse bool) error { +// processBlockProposal handles incoming block proposals. +// No errors are expected during normal operations. +func (e *Engine) processBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal) error { block := proposal.Block.ToInternal() header := block.Header + blockID := header.ID() - span, ctx := e.tracer.StartBlockSpan(context.Background(), header.ID(), trace.FollowerOnBlockProposal) + span, ctx := e.tracer.StartBlockSpan(context.Background(), blockID, trace.FollowerOnBlockProposal) defer span.End() log := e.log.With(). @@ -230,7 +244,7 @@ func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.Bl Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). - Hex("block_id", logging.Entity(header)). + Hex("block_id", blockID[:]). Hex("parent_id", header.ParentID[:]). Hex("payload_hash", header.PayloadHash[:]). Time("timestamp", header.Timestamp). @@ -247,14 +261,14 @@ func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.Bl // 3) blocks at a height below finalized height; they can not be finalized // ignore proposals that are already cached - _, cached := e.pending.ByID(header.ID()) + _, cached := e.pending.ByID(blockID) if cached { log.Debug().Msg("skipping already cached proposal") return nil } // ignore proposals that were already processed - _, err := e.headers.ByBlockID(header.ID()) + _, err := e.headers.ByBlockID(blockID) if err == nil { log.Debug().Msg("skipping already processed proposal") return nil @@ -276,6 +290,12 @@ func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.Bl Msg("dropping block too far ahead of locally finalized height") return nil } + if header.Height <= final.Height { + log.Debug(). + Uint64("final_height", final.Height). + Msg("dropping block below finalized threshold") + return nil + } // there are two possibilities if the proposal is neither already pending // processing in the cache, nor has already been processed: @@ -319,9 +339,9 @@ func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.Bl // at this point, we should be able to connect the proposal to the finalized // state and should process it to see whether to forward to hotstuff or not - err = e.processBlockAndDescendants(ctx, block, inRangeBlockResponse) + err = e.processBlockAndDescendants(ctx, block) if err != nil { - return fmt.Errorf("could not process block proposal: %w", err) + return fmt.Errorf("could not process block proposal (id=%x, height=%d, view=%d): %w", blockID, header.Height, header.View, err) } // most of the heavy database checks are done at this point, so this is a @@ -337,9 +357,8 @@ func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.Bl // The function assumes that `proposal` is connected to the finalized state. By induction, // any children are therefore also connected to the finalized state and can be processed as well. // No errors are expected during normal operations. -func (e *Engine) processBlockAndDescendants(ctx context.Context, block *flow.Block, inRangeBlockResponse bool) error { - header := block.Header - +func (e *Engine) processBlockAndDescendants(ctx context.Context, proposal *flow.Block) error { + header := proposal.Header span, ctx := e.tracer.StartSpanFromContext(ctx, trace.FollowerProcessBlockProposal) defer span.End() @@ -356,10 +375,36 @@ func (e *Engine) processBlockAndDescendants(ctx context.Context, block *flow.Blo log.Info().Msg("processing block proposal") + hotstuffProposal := model.ProposalFromFlow(header) + err := e.validator.ValidateProposal(hotstuffProposal) + if err != nil { + if model.IsInvalidBlockError(err) { + // TODO potential slashing + log.Err(err).Msgf("received invalid block proposal (potential slashing evidence)") + return nil + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We have received a proposal, but we don't know the epoch its view is within. + // We know: + // - the parent of this block is valid and inserted (ie. we knew the epoch for it) + // - if we then see this for the child, one of two things must have happened: + // 1. the proposer malicious created the block for a view very far in the future (it's invalid) + // -> in this case we can disregard the block + // 2. no blocks have been finalized the epoch commitment deadline, and the epoch end + // (breaking a critical assumption - see EpochCommitSafetyThreshold in protocol.Params for details) + // -> in this case, the network has encountered a critical failure + // - we assume in general that Case 2 will not happen, therefore we can discard this proposal + log.Err(err).Msg("unable to validate proposal with view from unknown epoch") + return nil + } + return fmt.Errorf("unexpected error validating proposal: %w", err) + } + // check whether the block is a valid extension of the chain. // it only checks the block header, since checking block body is expensive. // The full block check is done by the consensus participants. - err := e.state.Extend(ctx, block) + // TODO: CAUTION we write a block to disk, without validating its payload yet. This is vulnerable to malicious primaries. + err = e.state.Extend(ctx, proposal) if err != nil { // block is outdated by the time we started processing it // => some other node generating the proposal is probably behind is catching up. @@ -379,24 +424,13 @@ func (e *Engine) processBlockAndDescendants(ctx context.Context, block *flow.Blo return fmt.Errorf("could not extend protocol state: %w", err) } - // retrieve the parent - parent, err := e.headers.ByBlockID(header.ParentID) - if err != nil { - return fmt.Errorf("could not retrieve proposal parent: %w", err) - } - log.Info().Msg("forwarding block proposal to hotstuff") // submit the model to follower for processing - if inRangeBlockResponse { - <-e.follower.SubmitProposal(header, parent.View) - } else { - // ignore returned channel to avoid waiting - e.follower.SubmitProposal(header, parent.View) - } + e.follower.SubmitProposal(hotstuffProposal) // check for any descendants of the block to process - err = e.processPendingChildren(ctx, header, inRangeBlockResponse) + err = e.processPendingChildren(ctx, header) if err != nil { return fmt.Errorf("could not process pending children: %w", err) } @@ -407,7 +441,7 @@ func (e *Engine) processBlockAndDescendants(ctx context.Context, block *flow.Blo // processPendingChildren checks if there are proposals connected to the given // parent block that was just processed; if this is the case, they should now // all be validly connected to the finalized state and we should process them. -func (e *Engine) processPendingChildren(ctx context.Context, header *flow.Header, inRangeBlockResponse bool) error { +func (e *Engine) processPendingChildren(ctx context.Context, header *flow.Header) error { span, ctx := e.tracer.StartSpanFromContext(ctx, trace.FollowerProcessPendingChildren) defer span.End() @@ -423,13 +457,13 @@ func (e *Engine) processPendingChildren(ctx context.Context, header *flow.Header // then try to process children only this once var result *multierror.Error for _, child := range children { - err := e.processBlockAndDescendants(ctx, child.Message, inRangeBlockResponse) + err := e.processBlockAndDescendants(ctx, child.Message) if err != nil { result = multierror.Append(result, err) } } - // drop all of the children that should have been processed now + // drop all the children that should have been processed now e.pending.DropForParent(blockID) return result.ErrorOrNil() diff --git a/engine/common/follower/engine_test.go b/engine/common/follower/engine_test.go index f4faecb58f0..36a687e8c3b 100644 --- a/engine/common/follower/engine_test.go +++ b/engine/common/follower/engine_test.go @@ -1,17 +1,22 @@ package follower_test import ( + "context" "testing" + "time" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/module/trace" @@ -26,67 +31,90 @@ import ( type Suite struct { suite.Suite - net *mocknetwork.Network - con *mocknetwork.Conduit - me *module.Local - cleaner *storage.Cleaner - headers *storage.Headers - payloads *storage.Payloads - state *protocol.MutableState - snapshot *protocol.Snapshot - cache *module.PendingBlockBuffer - follower *module.HotStuffFollower - + net *mocknetwork.Network + con *mocknetwork.Conduit + me *module.Local + cleaner *storage.Cleaner + headers *storage.Headers + payloads *storage.Payloads + state *protocol.MutableState + snapshot *protocol.Snapshot + cache *module.PendingBlockBuffer + follower *module.HotStuffFollower + sync *module.BlockRequester + validator *hotstuff.Validator + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error engine *follower.Engine - sync *module.BlockRequester } -func (suite *Suite) SetupTest() { - - suite.net = new(mocknetwork.Network) - suite.con = new(mocknetwork.Conduit) - suite.me = new(module.Local) - suite.cleaner = new(storage.Cleaner) - suite.headers = new(storage.Headers) - suite.payloads = new(storage.Payloads) - suite.state = new(protocol.MutableState) - suite.snapshot = new(protocol.Snapshot) - suite.cache = new(module.PendingBlockBuffer) - suite.follower = new(module.HotStuffFollower) - suite.sync = new(module.BlockRequester) - - suite.net.On("Register", mock.Anything, mock.Anything).Return(suite.con, nil) - suite.cleaner.On("RunGC").Return() - suite.headers.On("Store", mock.Anything).Return(nil) - suite.payloads.On("Store", mock.Anything, mock.Anything).Return(nil) - suite.state.On("Final").Return(suite.snapshot) - suite.cache.On("PruneByView", mock.Anything).Return() - suite.cache.On("Size", mock.Anything).Return(uint(0)) +func (s *Suite) SetupTest() { + + s.net = mocknetwork.NewNetwork(s.T()) + s.con = mocknetwork.NewConduit(s.T()) + s.me = module.NewLocal(s.T()) + s.cleaner = storage.NewCleaner(s.T()) + s.headers = storage.NewHeaders(s.T()) + s.payloads = storage.NewPayloads(s.T()) + s.state = protocol.NewMutableState(s.T()) + s.snapshot = protocol.NewSnapshot(s.T()) + s.cache = module.NewPendingBlockBuffer(s.T()) + s.follower = module.NewHotStuffFollower(s.T()) + s.validator = hotstuff.NewValidator(s.T()) + s.sync = module.NewBlockRequester(s.T()) + + nodeID := unittest.IdentifierFixture() + s.me.On("NodeID").Return(nodeID).Maybe() + + s.net.On("Register", mock.Anything, mock.Anything).Return(s.con, nil) + s.cleaner.On("RunGC").Return().Maybe() + s.state.On("Final").Return(s.snapshot).Maybe() + s.cache.On("PruneByView", mock.Anything).Return().Maybe() + s.cache.On("Size", mock.Anything).Return(uint(0)).Maybe() metrics := metrics.NewNoopCollector() - eng, err := follower.New(zerolog.Logger{}, - suite.net, - suite.me, + eng, err := follower.New( + unittest.Logger(), + s.net, + s.me, metrics, metrics, - suite.cleaner, - suite.headers, - suite.payloads, - suite.state, - suite.cache, - suite.follower, - suite.sync, + s.cleaner, + s.headers, + s.payloads, + s.state, + s.cache, + s.follower, + s.validator, + s.sync, trace.NewNoopTracer()) - require.Nil(suite.T(), err) + require.Nil(s.T(), err) + + s.engine = eng + + s.ctx, s.cancel, s.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + s.engine.Start(s.ctx) + unittest.RequireCloseBefore(s.T(), s.engine.Ready(), time.Second, "engine failed to start") +} - suite.engine = eng +// TearDownTest stops the engine and checks there are no errors thrown to the SignallerContext. +func (s *Suite) TearDownTest() { + s.cancel() + unittest.RequireCloseBefore(s.T(), s.engine.Done(), time.Second, "engine failed to stop") + select { + case err := <-s.errs: + assert.NoError(s.T(), err) + default: + } } func TestFollower(t *testing.T) { suite.Run(t, new(Suite)) } -func (suite *Suite) TestHandlePendingBlock() { +func (s *Suite) TestHandlePendingBlock() { originID := unittest.IdentifierFixture() head := unittest.BlockFixture() @@ -96,28 +124,30 @@ func (suite *Suite) TestHandlePendingBlock() { block.Header.Height = 12 // not in cache - suite.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + s.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() + s.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() // don't return the parent when requested - suite.snapshot.On("Head").Return(head.Header, nil) - suite.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.headers.On("ByBlockID", block.Header.ParentID).Return(nil, realstorage.ErrNotFound).Once() + s.snapshot.On("Head").Return(head.Header, nil) + s.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() + s.headers.On("ByBlockID", block.Header.ParentID).Return(nil, realstorage.ErrNotFound).Once() - suite.cache.On("Add", mock.Anything, mock.Anything).Return(true).Once() - suite.sync.On("RequestBlock", block.Header.ParentID, block.Header.Height-1).Return().Once() + done := make(chan struct{}) + s.cache.On("Add", mock.Anything, mock.Anything).Return(true).Once() + s.sync.On("RequestBlock", block.Header.ParentID, block.Header.Height-1).Run(func(_ mock.Arguments) { + close(done) + }).Return().Once() // submit the block proposal := unittest.ProposalFromBlock(&block) - err := suite.engine.Process(channels.ReceiveBlocks, originID, proposal) - assert.Nil(suite.T(), err) + err := s.engine.Process(channels.ReceiveBlocks, originID, proposal) + assert.Nil(s.T(), err) - suite.follower.AssertNotCalled(suite.T(), "SubmitProposal", mock.Anything) - suite.cache.AssertExpectations(suite.T()) - suite.con.AssertExpectations(suite.T()) + unittest.AssertClosesBefore(s.T(), done, time.Second) + s.follower.AssertNotCalled(s.T(), "SubmitProposal", mock.Anything) } -func (suite *Suite) TestHandleProposal() { +func (s *Suite) TestHandleProposal() { originID := unittest.IdentifierFixture() parent := unittest.BlockFixture() @@ -128,103 +158,154 @@ func (suite *Suite) TestHandleProposal() { block.Header.ParentID = parent.ID() // not in cache - suite.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + s.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() + s.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() + s.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + + done := make(chan struct{}) + hotstuffProposal := model.ProposalFromFlow(block.Header) // the parent is the last finalized state - suite.snapshot.On("Head").Return(parent.Header, nil) + s.snapshot.On("Head").Return(parent.Header, nil) + // the block passes hotstuff validation + s.validator.On("ValidateProposal", hotstuffProposal).Return(nil) // we should be able to extend the state with the block - suite.state.On("Extend", mock.Anything, &block).Return(nil).Once() + s.state.On("Extend", mock.Anything, &block).Return(nil).Once() // we should be able to get the parent header by its ID - suite.headers.On("ByBlockID", block.Header.ParentID).Return(parent.Header, nil).Twice() + s.headers.On("ByBlockID", block.Header.ParentID).Return(parent.Header, nil).Once() // we do not have any children cached - suite.cache.On("ByParentID", block.ID()).Return(nil, false) + s.cache.On("ByParentID", block.ID()).Return(nil, false) // the proposal should be forwarded to the follower - suite.follower.On("SubmitProposal", block.Header, parent.Header.View).Once().Return(make(<-chan struct{})) + s.follower.On("SubmitProposal", hotstuffProposal).Run(func(_ mock.Arguments) { + close(done) + }).Once() // submit the block proposal := unittest.ProposalFromBlock(&block) - err := suite.engine.Process(channels.ReceiveBlocks, originID, proposal) - assert.Nil(suite.T(), err) - - suite.follower.AssertExpectations(suite.T()) + err := s.engine.Process(channels.ReceiveBlocks, originID, proposal) + assert.Nil(s.T(), err) + unittest.AssertClosesBefore(s.T(), done, time.Second) } -func (suite *Suite) TestHandleProposalSkipProposalThreshold() { +func (s *Suite) TestHandleProposalSkipProposalThreshold() { // mock latest finalized state final := unittest.BlockHeaderFixture() - suite.snapshot.On("Head").Return(final, nil) + s.snapshot.On("Head").Return(final, nil) originID := unittest.IdentifierFixture() block := unittest.BlockFixture() block.Header.Height = final.Height + compliance.DefaultConfig().SkipNewProposalsThreshold + 1 + done := make(chan struct{}) + // not in cache or storage - suite.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + s.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() + s.headers.On("ByBlockID", block.ID()).Run(func(_ mock.Arguments) { + close(done) + }).Return(nil, realstorage.ErrNotFound).Once() // submit the block proposal := unittest.ProposalFromBlock(&block) - err := suite.engine.Process(channels.ReceiveBlocks, originID, proposal) - assert.NoError(suite.T(), err) + err := s.engine.Process(channels.ReceiveBlocks, originID, proposal) + assert.NoError(s.T(), err) + unittest.AssertClosesBefore(s.T(), done, time.Second) // block should be dropped - not added to state or cache - suite.state.AssertNotCalled(suite.T(), "Extend", mock.Anything) - suite.cache.AssertNotCalled(suite.T(), "Add", originID, mock.Anything) + s.state.AssertNotCalled(s.T(), "Extend", mock.Anything) + s.cache.AssertNotCalled(s.T(), "Add", originID, mock.Anything) } -func (suite *Suite) TestHandleProposalWithPendingChildren() { +// TestHandleProposalWithPendingChildren tests processing a block which has a pending +// child cached. +// - the block should be processed +// - the cached child block should also be processed +func (s *Suite) TestHandleProposalWithPendingChildren() { originID := unittest.IdentifierFixture() - parent := unittest.BlockFixture() - block := unittest.BlockFixture() - child := unittest.BlockFixture() + parent := unittest.BlockFixture() // already processed and incorporated block + block := unittest.BlockWithParentFixture(parent.Header) // block which is passed as input to the engine + child := unittest.BlockWithParentFixture(block.Header) // block which is already cached - parent.Header.Height = 9 - block.Header.Height = 10 - child.Header.Height = 11 - - block.Header.ParentID = parent.ID() - child.Header.ParentID = block.ID() + done := make(chan struct{}) + hotstuffProposal := model.ProposalFromFlow(block.Header) + childHotstuffProposal := model.ProposalFromFlow(child.Header) // the parent is the last finalized state - suite.snapshot.On("Head").Return(parent.Header, nil).Once() - suite.snapshot.On("Head").Return(block.Header, nil).Once() + s.snapshot.On("Head").Return(parent.Header, nil) - // both parent and child not in cache - // suite.cache.On("ByID", child.ID()).Return(nil, false).Once() - suite.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() - suite.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() + s.cache.On("ByID", mock.Anything).Return(flow.Slashable[flow.Block]{}, false) // first time calling, assume it's not there - suite.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() - // should extend state with new block - suite.state.On("Extend", mock.Anything, &block).Return(nil).Once() - suite.state.On("Extend", mock.Anything, &child).Return(nil).Once() + s.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + // both blocks pass HotStuff validation + s.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + s.validator.On("ValidateProposal", childHotstuffProposal).Return(nil) + // should extend state with the input block, and the child + s.state.On("Extend", mock.Anything, block).Return(nil).Once() + s.state.On("Extend", mock.Anything, child).Return(nil).Once() // we have already received and stored the parent - suite.headers.On("ByBlockID", parent.ID()).Return(parent.Header, nil) - suite.headers.On("ByBlockID", block.ID()).Return(block.Header, nil).Once() + s.headers.On("ByBlockID", parent.ID()).Return(parent.Header, nil).Once() // should submit to follower - suite.follower.On("SubmitProposal", block.Header, parent.Header.View).Once().Return(make(<-chan struct{})) - suite.follower.On("SubmitProposal", child.Header, block.Header.View).Once().Return(make(<-chan struct{})) + s.follower.On("SubmitProposal", hotstuffProposal).Once() + s.follower.On("SubmitProposal", childHotstuffProposal).Run(func(_ mock.Arguments) { + close(done) + }).Once() // we have one pending child cached pending := []flow.Slashable[flow.Block]{ { OriginID: originID, - Message: &child, + Message: child, }, } - suite.cache.On("ByParentID", block.ID()).Return(pending, true) - suite.cache.On("ByParentID", child.ID()).Return(nil, false) - suite.cache.On("DropForParent", block.ID()).Once() + s.cache.On("ByParentID", block.ID()).Return(pending, true).Once() + s.cache.On("ByParentID", child.ID()).Return(nil, false).Once() + s.cache.On("DropForParent", block.ID()).Once() // submit the block proposal - proposal := unittest.ProposalFromBlock(&block) - err := suite.engine.Process(channels.ReceiveBlocks, originID, proposal) - assert.Nil(suite.T(), err) + proposal := unittest.ProposalFromBlock(block) + err := s.engine.Process(channels.ReceiveBlocks, originID, proposal) + assert.NoError(s.T(), err) + unittest.AssertClosesBefore(s.T(), done, time.Second) +} - suite.follower.AssertExpectations(suite.T()) +// TestProcessSyncedBlock checks if processing synced block using unsafe API results in error. +// All blocks from sync engine should be sent through dedicated compliance API. +func (s *Suite) TestProcessSyncedBlock() { + parent := unittest.BlockFixture() + block := unittest.BlockFixture() + + parent.Header.Height = 10 + block.Header.Height = 11 + block.Header.ParentID = parent.ID() + + // not in cache + s.cache.On("ByID", block.ID()).Return(flow.Slashable[flow.Block]{}, false).Once() + s.cache.On("ByID", block.Header.ParentID).Return(flow.Slashable[flow.Block]{}, false).Once() + s.headers.On("ByBlockID", block.ID()).Return(nil, realstorage.ErrNotFound).Once() + + done := make(chan struct{}) + hotstuffProposal := model.ProposalFromFlow(block.Header) + + // the parent is the last finalized state + s.snapshot.On("Head").Return(parent.Header, nil) + // the block passes hotstuff validation + s.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + // we should be able to extend the state with the block + s.state.On("Extend", mock.Anything, &block).Return(nil).Once() + // we should be able to get the parent header by its ID + s.headers.On("ByBlockID", block.Header.ParentID).Return(parent.Header, nil).Once() + // we do not have any children cached + s.cache.On("ByParentID", block.ID()).Return(nil, false) + // the proposal should be forwarded to the follower + s.follower.On("SubmitProposal", hotstuffProposal).Run(func(_ mock.Arguments) { + close(done) + }).Once() + + s.engine.OnSyncedBlock(flow.Slashable[messages.BlockProposal]{ + OriginID: unittest.IdentifierFixture(), + Message: messages.NewBlockProposal(&block), + }) + unittest.AssertClosesBefore(s.T(), done, time.Second) } diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index 542eaf33d85..f1b698e6b11 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/onflow/flow/protobuf/go/flow/entities" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" "github.com/onflow/flow-go/crypto" @@ -143,6 +145,22 @@ func BlockHeaderToMessage(h *flow.Header, signerIDs flow.IdentifierList) (*entit id := h.ID() t := timestamppb.New(h.Timestamp) + var lastViewTC *entities.TimeoutCertificate + if h.LastViewTC != nil { + newestQC := h.LastViewTC.NewestQC + lastViewTC = &entities.TimeoutCertificate{ + View: h.LastViewTC.View, + HighQcViews: h.LastViewTC.NewestQCViews, + SignerIndices: h.LastViewTC.SignerIndices, + SigData: h.LastViewTC.SigData, + HighestQc: &entities.QuorumCertificate{ + View: newestQC.View, + BlockId: newestQC.BlockID[:], + SignerIndices: newestQC.SignerIndices, + SigData: newestQC.SigData, + }, + } + } parentVoterIds := IdentifiersToMessages(signerIDs) return &entities.BlockHeader{ @@ -152,12 +170,14 @@ func BlockHeaderToMessage(h *flow.Header, signerIDs flow.IdentifierList) (*entit PayloadHash: h.PayloadHash[:], Timestamp: t, View: h.View, + ParentView: h.ParentView, ParentVoterIndices: h.ParentVoterIndices, ParentVoterIds: parentVoterIds, ParentVoterSigData: h.ParentVoterSigData, ProposerId: h.ProposerID[:], ProposerSigData: h.ProposerSigData, ChainId: h.ChainID.String(), + LastViewTc: lastViewTC, }, nil } @@ -166,17 +186,39 @@ func MessageToBlockHeader(m *entities.BlockHeader) (*flow.Header, error) { if err != nil { return nil, fmt.Errorf("failed to convert ChainId: %w", err) } + var lastViewTC *flow.TimeoutCertificate + if m.LastViewTc != nil { + newestQC := m.LastViewTc.HighestQc + if newestQC == nil { + return nil, fmt.Errorf("invalid structure newest QC should be present") + } + lastViewTC = &flow.TimeoutCertificate{ + View: m.LastViewTc.View, + NewestQCViews: m.LastViewTc.HighQcViews, + SignerIndices: m.LastViewTc.SignerIndices, + SigData: m.LastViewTc.SigData, + NewestQC: &flow.QuorumCertificate{ + View: newestQC.View, + BlockID: MessageToIdentifier(newestQC.BlockId), + SignerIndices: newestQC.SignerIndices, + SigData: newestQC.SigData, + }, + } + } + return &flow.Header{ ParentID: MessageToIdentifier(m.ParentId), Height: m.Height, PayloadHash: MessageToIdentifier(m.PayloadHash), Timestamp: m.Timestamp.AsTime(), View: m.View, + ParentView: m.ParentView, ParentVoterIndices: m.ParentVoterIndices, ParentVoterSigData: m.ParentVoterSigData, ProposerID: MessageToIdentifier(m.ProposerId), ProposerSigData: m.ProposerSigData, ChainID: *chainId, + LastViewTC: lastViewTC, }, nil } @@ -845,34 +887,37 @@ func ChunkExecutionDataToMessage(data *execution_data.ChunkExecutionData) (*enti events = nil } - paths := make([][]byte, len(data.TrieUpdate.Paths)) - for i, path := range data.TrieUpdate.Paths { - paths[i] = path[:] - } - - payloads := make([]*entities.Payload, len(data.TrieUpdate.Payloads)) - for i, payload := range data.TrieUpdate.Payloads { - key, err := payload.Key() - if err != nil { - return nil, err + var trieUpdate *entities.TrieUpdate + if data.TrieUpdate != nil { + paths := make([][]byte, len(data.TrieUpdate.Paths)) + for i, path := range data.TrieUpdate.Paths { + paths[i] = path[:] } - keyParts := make([]*entities.KeyPart, len(key.KeyParts)) - for j, keyPart := range key.KeyParts { - keyParts[j] = &entities.KeyPart{ - Type: uint32(keyPart.Type), - Value: keyPart.Value, + + payloads := make([]*entities.Payload, len(data.TrieUpdate.Payloads)) + for i, payload := range data.TrieUpdate.Payloads { + key, err := payload.Key() + if err != nil { + return nil, err + } + keyParts := make([]*entities.KeyPart, len(key.KeyParts)) + for j, keyPart := range key.KeyParts { + keyParts[j] = &entities.KeyPart{ + Type: uint32(keyPart.Type), + Value: keyPart.Value, + } + } + payloads[i] = &entities.Payload{ + KeyPart: keyParts, + Value: payload.Value(), } } - payloads[i] = &entities.Payload{ - KeyPart: keyParts, - Value: payload.Value(), - } - } - trieUpdate := &entities.TrieUpdate{ - RootHash: data.TrieUpdate.RootHash[:], - Paths: paths, - Payloads: payloads, + trieUpdate = &entities.TrieUpdate{ + RootHash: data.TrieUpdate.RootHash[:], + Paths: paths, + Payloads: payloads, + } } return &entities.ChunkExecutionData{ @@ -902,14 +947,17 @@ func MessageToBlockExecutionData(m *entities.BlockExecutionData, chain flow.Chai } func MessageToChunkExecutionData(m *entities.ChunkExecutionData, chain flow.Chain) (*execution_data.ChunkExecutionData, error) { - collection, err := messageToExecutionDataCollection(m.GetCollection(), chain) + collection, err := messageToTrustedCollection(m.GetCollection(), chain) if err != nil { return nil, err } - trieUpdate, err := messageToTrieUpdate(m.GetTrieUpdate()) - if err != nil { - return nil, err + var trieUpdate *ledger.TrieUpdate + if m.GetTrieUpdate() != nil { + trieUpdate, err = MessageToTrieUpdate(m.GetTrieUpdate()) + if err != nil { + return nil, err + } } events := MessagesToEvents(m.GetEvents()) @@ -924,13 +972,13 @@ func MessageToChunkExecutionData(m *entities.ChunkExecutionData, chain flow.Chai }, nil } -func messageToExecutionDataCollection(m *entities.ExecutionDataCollection, chain flow.Chain) (*flow.Collection, error) { +func messageToTrustedCollection(m *entities.ExecutionDataCollection, chain flow.Chain) (*flow.Collection, error) { messages := m.GetTransactions() transactions := make([]*flow.TransactionBody, len(messages)) for i, message := range messages { - transaction, err := MessageToTransaction(message, chain) + transaction, err := messageToTrustedTransaction(message, chain) if err != nil { - return nil, err + return nil, fmt.Errorf("could not convert transaction %d: %w", i, err) } transactions[i] = &transaction } @@ -942,17 +990,77 @@ func messageToExecutionDataCollection(m *entities.ExecutionDataCollection, chain return &flow.Collection{Transactions: transactions}, nil } -func messageToTrieUpdate(m *entities.TrieUpdate) (*ledger.TrieUpdate, error) { +// messageToTrustedTransaction converts a transaction message to a transaction body. +// This is useful when converting transactions from trusted state like BlockExecutionData which +// contain service transactions that do not conform to external transaction format. +func messageToTrustedTransaction(m *entities.Transaction, chain flow.Chain) (flow.TransactionBody, error) { + if m == nil { + return flow.TransactionBody{}, ErrEmptyMessage + } + + t := flow.NewTransactionBody() + + proposalKey := m.GetProposalKey() + if proposalKey != nil { + proposalAddress, err := insecureAddress(proposalKey.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert proposer address: %w", err) + } + t.SetProposalKey(proposalAddress, uint64(proposalKey.GetKeyId()), proposalKey.GetSequenceNumber()) + } + + payer := m.GetPayer() + if payer != nil { + payerAddress, err := insecureAddress(payer, chain) + if err != nil { + return *t, fmt.Errorf("could not convert payer address: %w", err) + } + t.SetPayer(payerAddress) + } + + for _, authorizer := range m.GetAuthorizers() { + authorizerAddress, err := Address(authorizer, chain) + if err != nil { + return *t, fmt.Errorf("could not convert authorizer address: %w", err) + } + t.AddAuthorizer(authorizerAddress) + } + + for _, sig := range m.GetPayloadSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert payload signature address: %w", err) + } + t.AddPayloadSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + for _, sig := range m.GetEnvelopeSignatures() { + addr, err := Address(sig.GetAddress(), chain) + if err != nil { + return *t, fmt.Errorf("could not convert envelope signature address: %w", err) + } + t.AddEnvelopeSignature(addr, uint64(sig.GetKeyId()), sig.GetSignature()) + } + + t.SetScript(m.GetScript()) + t.SetArguments(m.GetArguments()) + t.SetReferenceBlockID(flow.HashToID(m.GetReferenceBlockId())) + t.SetGasLimit(m.GetGasLimit()) + + return *t, nil +} + +func MessageToTrieUpdate(m *entities.TrieUpdate) (*ledger.TrieUpdate, error) { rootHash, err := ledger.ToRootHash(m.GetRootHash()) if err != nil { - return nil, err + return nil, fmt.Errorf("could not convert root hash: %w", err) } paths := make([]ledger.Path, len(m.GetPaths())) for i, path := range m.GetPaths() { convertedPath, err := ledger.ToPath(path) if err != nil { - return nil, err + return nil, fmt.Errorf("could not convert path %d: %w", i, err) } paths[i] = convertedPath } @@ -972,3 +1080,14 @@ func messageToTrieUpdate(m *entities.TrieUpdate) (*ledger.TrieUpdate, error) { Payloads: payloads, }, nil } + +// insecureAddress converts a raw address to a flow.Address, skipping validation +// This is useful when converting transactions from trusted state like BlockExecutionData. +// This should only be used for trusted inputs +func insecureAddress(rawAddress []byte, chain flow.Chain) (flow.Address, error) { + if len(rawAddress) == 0 { + return flow.EmptyAddress, status.Error(codes.InvalidArgument, "address cannot be empty") + } + + return flow.BytesToAddress(rawAddress), nil +} diff --git a/engine/common/rpc/convert/convert_test.go b/engine/common/rpc/convert/convert_test.go index e4129db57b4..a98f828d0f6 100644 --- a/engine/common/rpc/convert/convert_test.go +++ b/engine/common/rpc/convert/convert_test.go @@ -84,6 +84,18 @@ func TestConvertBlockExecutionData(t *testing.T) { // Fill the chunk execution datas with trie updates, collections, and events minSerializedSize := uint64(10 * execution_data.DefaultMaxBlobSize) for i := 0; i < numChunks; i++ { + // the service chunk sometimes does not have any trie updates + if i == numChunks-1 { + tx1 := unittest.TransactionBodyFixture() + // proposal key and payer are empty addresses for service tx + tx1.ProposalKey.Address = flow.EmptyAddress + tx1.Payer = flow.EmptyAddress + bed.ChunkExecutionDatas[i] = &execution_data.ChunkExecutionData{ + Collection: &flow.Collection{Transactions: []*flow.TransactionBody{&tx1}}, + } + continue + } + // Initialize collection tx1 := unittest.TransactionBodyFixture() tx2 := unittest.TransactionBodyFixture() diff --git a/engine/common/rpc/errors.go b/engine/common/rpc/errors.go index cc7bcd05f94..0e8af80317b 100644 --- a/engine/common/rpc/errors.go +++ b/engine/common/rpc/errors.go @@ -3,12 +3,15 @@ package rpc import ( "errors" + "github.com/hashicorp/go-multierror" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/onflow/flow-go/storage" ) +// ConvertStorageError converts a generic error into a grpc status error, converting storage errors +// into codes.NotFound func ConvertStorageError(err error) error { if err == nil { return nil @@ -24,3 +27,48 @@ func ConvertStorageError(err error) error { return status.Errorf(codes.Internal, "failed to find: %v", err) } + +// ConvertMultiError converts a multierror to a grpc status error. +// If the errors have related status codes, the common code is returned, otherwise defaultCode is used. +func ConvertMultiError(err *multierror.Error, msg string, defaultCode codes.Code) error { + allErrors := err.WrappedErrors() + if len(allErrors) == 0 { + return nil + } + + if msg != "" { + msg += ": " + } + + // get a list of all of status codes + allCodes := make(map[codes.Code]struct{}) + for _, err := range allErrors { + allCodes[status.Code(err)] = struct{}{} + } + + // if they all match, return that + if len(allCodes) == 1 { + code := status.Code(allErrors[0]) + return status.Errorf(code, "%s%v", msg, err) + } + + // if they mostly match, ignore Unavailable and DeadlineExceeded since any other code is + // more descriptive + if len(allCodes) == 2 { + if _, ok := allCodes[codes.Unavailable]; ok { + delete(allCodes, codes.Unavailable) + for code := range allCodes { + return status.Errorf(code, "%s%v", msg, err) + } + } + if _, ok := allCodes[codes.DeadlineExceeded]; ok { + delete(allCodes, codes.DeadlineExceeded) + for code := range allCodes { + return status.Errorf(code, "%s%v", msg, err) + } + } + } + + // otherwise, return the default code + return status.Errorf(defaultCode, "%s%v", msg, err) +} diff --git a/engine/common/rpc/errors_test.go b/engine/common/rpc/errors_test.go new file mode 100644 index 00000000000..e35dc262245 --- /dev/null +++ b/engine/common/rpc/errors_test.go @@ -0,0 +1,73 @@ +package rpc + +import ( + "fmt" + "testing" + + "github.com/hashicorp/go-multierror" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "gotest.tools/assert" +) + +func TestConvertMultiError(t *testing.T) { + defaultCode := codes.Internal + t.Run("single error", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + + err := ConvertMultiError(errors, "", defaultCode) + assert.Equal(t, codes.NotFound, status.Code(err)) + }) + + t.Run("same code", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + + err := ConvertMultiError(errors, "", defaultCode) + assert.Equal(t, codes.NotFound, status.Code(err)) + }) + + t.Run("same codes - unavailable ignored", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, status.Error(codes.Unavailable, "unavailable")) + errors = multierror.Append(errors, status.Error(codes.Unavailable, "unavailable")) + + err := ConvertMultiError(errors, "", defaultCode) + assert.Equal(t, codes.NotFound, status.Code(err)) + }) + + t.Run("same codes - deadline exceeded ignored", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.DeadlineExceeded, "deadline exceeded")) + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + + err := ConvertMultiError(errors, "", defaultCode) + assert.Equal(t, codes.NotFound, status.Code(err)) + }) + + t.Run("all different codes", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, status.Error(codes.Internal, "internal")) + errors = multierror.Append(errors, status.Error(codes.InvalidArgument, "invalid arg")) + + err := ConvertMultiError(errors, "", defaultCode) + assert.Equal(t, defaultCode, status.Code(err)) + }) + + t.Run("non-grpc errors", func(t *testing.T) { + var errors *multierror.Error + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + errors = multierror.Append(errors, fmt.Errorf("not a grpc status code")) + errors = multierror.Append(errors, status.Error(codes.NotFound, "not found")) + + err := ConvertMultiError(errors, "some prefix", defaultCode) + assert.Equal(t, defaultCode, status.Code(err)) + assert.ErrorContains(t, err, "some prefix: ") + }) +} diff --git a/engine/common/synchronization/engine.go b/engine/common/synchronization/engine.go index b5b7f9dd6b3..e31880e30e0 100644 --- a/engine/common/synchronization/engine.go +++ b/engine/common/synchronization/engine.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" + "github.com/onflow/flow-go/engine/consensus" "github.com/onflow/flow-go/model/chainsync" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" @@ -39,7 +40,7 @@ type Engine struct { me module.Local con network.Conduit blocks storage.Blocks - comp network.Engine // compliance layer engine + comp consensus.Compliance pollInterval time.Duration scanInterval time.Duration @@ -61,7 +62,7 @@ func New( net network.Network, me module.Local, blocks storage.Blocks, - comp network.Engine, + comp consensus.Compliance, core module.SyncCore, finalizedHeader *FinalizedHeaderCache, participantsProvider module.IdentifierProvider, @@ -300,13 +301,19 @@ func (e *Engine) onBlockResponse(originID flow.Identifier, res *messages.BlockRe e.log.Debug().Uint64("first", first).Uint64("last", last).Msg("received block response") for _, block := range res.Blocks { - if !e.core.HandleBlock(&block.Header) { - e.log.Debug().Uint64("height", block.Header.Height).Msg("block handler rejected") + header := block.Header + if !e.core.HandleBlock(&header) { + e.log.Debug().Uint64("height", header.Height).Msg("block handler rejected") continue } + // forward the block to the compliance engine for validation and processing + e.comp.OnSyncedBlock(flow.Slashable[messages.BlockProposal]{ + OriginID: originID, + Message: &messages.BlockProposal{ + Block: block, + }, + }) } - - e.comp.SubmitLocal(res) } // checkLoop will regularly scan for items that need requesting. diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index 59344cb7031..c38e101484f 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -15,7 +15,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/model/events" + mockconsensus "github.com/onflow/flow-go/engine/consensus/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" @@ -52,7 +52,7 @@ type SyncSuite struct { state *protocol.State snapshot *protocol.Snapshot blocks *storage.Blocks - comp *mocknetwork.Engine + comp *mockconsensus.Compliance core *module.SyncCore e *Engine } @@ -158,8 +158,8 @@ func (ss *SyncSuite) SetupTest() { ) // set up compliance engine mock - ss.comp = &mocknetwork.Engine{} - ss.comp.On("SubmitLocal", mock.Anything).Return() + ss.comp = mockconsensus.NewCompliance(ss.T()) + ss.comp.On("Process", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() // set up sync core ss.core = &module.SyncCore{} @@ -354,7 +354,7 @@ func (ss *SyncSuite) TestOnRangeRequest() { var err error config := synccore.DefaultConfig() config.MaxSize = 2 - ss.e.requestHandler.core, err = synccore.New(ss.e.log, config, metrics.NewNoopCollector()) + ss.e.requestHandler.core, err = synccore.New(ss.e.log, config, metrics.NewNoopCollector(), flow.Localnet) require.NoError(ss.T(), err) err = ss.e.requestHandler.onRangeRequest(originID, req) @@ -431,7 +431,7 @@ func (ss *SyncSuite) TestOnBatchRequest() { var err error config := synccore.DefaultConfig() config.MaxSize = 2 - ss.e.requestHandler.core, err = synccore.New(ss.e.log, config, metrics.NewNoopCollector()) + ss.e.requestHandler.core, err = synccore.New(ss.e.log, config, metrics.NewNoopCollector(), flow.Localnet) require.NoError(ss.T(), err) err = ss.e.requestHandler.onBatchRequest(originID, req) @@ -458,15 +458,15 @@ func (ss *SyncSuite) TestOnBlockResponse() { ss.core.On("HandleBlock", unprocessable.Header).Return(false) res.Blocks = append(res.Blocks, messages.UntrustedBlockFromInternal(&unprocessable)) - ss.comp.On("SubmitLocal", mock.Anything).Run(func(args mock.Arguments) { - res := args.Get(0).(*events.SyncedBlock) - ss.Assert().Equal(&processable, res.Block) + ss.comp.On("OnSyncedBlock", mock.Anything).Run(func(args mock.Arguments) { + res := args.Get(0).(flow.Slashable[messages.BlockProposal]) + converted := res.Message.Block.ToInternal() + ss.Assert().Equal(processable.Header, converted.Header) + ss.Assert().Equal(processable.Payload, converted.Payload) ss.Assert().Equal(originID, res.OriginID) - }, - ) + }) ss.e.onBlockResponse(originID, res) - ss.comp.AssertExpectations(ss.T()) ss.core.AssertExpectations(ss.T()) } diff --git a/engine/consensus/approvals/request_tracker.go b/engine/consensus/approvals/request_tracker.go index f4dbb4f3a6a..02520d10ee7 100644 --- a/engine/consensus/approvals/request_tracker.go +++ b/engine/consensus/approvals/request_tracker.go @@ -86,7 +86,7 @@ func NewRequestTracker(headers storage.Headers, blackoutPeriodMin, blackoutPerio // TryUpdate tries to update tracker item if it's not in blackout period. Returns the tracker item for a specific chunk // (creates it if it doesn't exists) and whenever request item was successfully updated or not. // Since RequestTracker prunes items by height it can't accept items for height lower than cached lowest height. -// If height of executed block pointed by execution result is smaller than the lowest height, sentinel mempool.DecreasingPruningHeightError is returned. +// If height of executed block pointed by execution result is smaller than the lowest height, sentinel mempool.BelowPrunedThresholdError is returned. // In case execution result points to unknown executed block exception will be returned. func (rt *RequestTracker) TryUpdate(result *flow.ExecutionResult, incorporatedBlockID flow.Identifier, chunkIndex uint64) (RequestTrackerItem, bool, error) { resultID := result.ID() @@ -119,7 +119,7 @@ func (rt *RequestTracker) set(resultID, executedBlockID, incorporatedBlockID flo } if executedBlock.Height < rt.lowestHeight { - return mempool.NewDecreasingPruningHeightErrorf( + return mempool.NewBelowPrunedThresholdErrorf( "adding height: %d, existing height: %d", executedBlock.Height, rt.lowestHeight) } @@ -171,12 +171,12 @@ func (rt *RequestTracker) Remove(resultIDs ...flow.Identifier) { // Monotonicity Requirement: // The pruned height cannot decrease, as we cannot recover already pruned elements. // If `height` is smaller than the previous value, the previous value is kept -// and the sentinel mempool.DecreasingPruningHeightError is returned. +// and the sentinel mempool.BelowPrunedThresholdError is returned. func (rt *RequestTracker) PruneUpToHeight(height uint64) error { rt.lock.Lock() defer rt.lock.Unlock() if height < rt.lowestHeight { - return mempool.NewDecreasingPruningHeightErrorf( + return mempool.NewBelowPrunedThresholdErrorf( "pruning height: %d, existing height: %d", height, rt.lowestHeight) } diff --git a/engine/consensus/approvals/request_tracker_test.go b/engine/consensus/approvals/request_tracker_test.go index 5a1e017ba9a..33fe47e708c 100644 --- a/engine/consensus/approvals/request_tracker_test.go +++ b/engine/consensus/approvals/request_tracker_test.go @@ -111,7 +111,7 @@ func (s *RequestTrackerTestSuite) TestTryUpdate_UpdateForPrunedHeight() { result := unittest.ExecutionResultFixture(unittest.WithBlock(&executedBlock)) _, updated, err := s.tracker.TryUpdate(result, executedBlock.ID(), uint64(0)) require.Error(s.T(), err) - require.True(s.T(), mempool.IsDecreasingPruningHeightError(err)) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) require.False(s.T(), updated) } @@ -140,12 +140,12 @@ func (s *RequestTrackerTestSuite) TestPruneUpToHeight_Pruning() { require.True(s.T(), ok) } -// TestPruneUpToHeight_PruningWithDecreasingHeight tests that pruning with decreasing height results in error -func (s *RequestTrackerTestSuite) TestPruneUpToHeight_PruningWithDecreasingHeight() { +// TestPruneUpToHeight_PruningWithHeightBelowPrunedThreshold tests that pruning with height below pruned threshold results in error +func (s *RequestTrackerTestSuite) TestPruneUpToHeight_PruningWithHeightBelowPrunedThreshold() { height := uint64(100) err := s.tracker.PruneUpToHeight(height + 1) require.NoError(s.T(), err) err = s.tracker.PruneUpToHeight(height) require.Error(s.T(), err) - require.True(s.T(), mempool.IsDecreasingPruningHeightError(err)) + require.True(s.T(), mempool.IsBelowPrunedThresholdError(err)) } diff --git a/engine/consensus/approvals/verifying_assignment_collector.go b/engine/consensus/approvals/verifying_assignment_collector.go index 71f0f11ff7b..118627db3bc 100644 --- a/engine/consensus/approvals/verifying_assignment_collector.go +++ b/engine/consensus/approvals/verifying_assignment_collector.go @@ -340,7 +340,7 @@ func (ac *VerifyingAssignmentCollector) RequestMissingApprovals(observation cons if err != nil { // it could happen that other gorotuine will prune request tracker because of sealing progress // in this case we should just stop requesting approvals as block was already sealed - if mempool.IsDecreasingPruningHeightError(err) { + if mempool.IsBelowPrunedThresholdError(err) { return 0, nil } return 0, err diff --git a/engine/consensus/compliance.go b/engine/consensus/compliance.go new file mode 100644 index 00000000000..3630eba084e --- /dev/null +++ b/engine/consensus/compliance.go @@ -0,0 +1,30 @@ +package consensus + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/component" +) + +// Compliance defines the interface to the consensus logic that precedes hotstuff logic. +// It's responsible for processing incoming block proposals broadcast by other consensus nodes +// as well as blocks obtained via the sync protocol. +// Compliance logic performs validation of incoming blocks depending on internal implementation. +// Main consensus logic performs full validation by checking headers and payloads. +// Follower consensus logic checks header validity and by observing a valid QC can make a statement about +// payload validity of parent block. +// Compliance logic guarantees that only valid blocks are added to chain state, passed to hotstuff and other +// components. +// Implementation need to be non-blocking and concurrency safe. +type Compliance interface { + component.Component + + // OnBlockProposal feeds a new block proposal into the processing pipeline. + // Incoming proposals will be queued and eventually dispatched by worker. + // This method is non-blocking. + OnBlockProposal(proposal flow.Slashable[messages.BlockProposal]) + // OnSyncedBlock feeds a block obtained from sync proposal into the processing pipeline. + // Incoming proposals will be queued and eventually dispatched by worker. + // This method is non-blocking. + OnSyncedBlock(block flow.Slashable[messages.BlockProposal]) +} diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index da2ba413aa5..8f6a11c0eb3 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -14,10 +14,12 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/consensus/sealing/counters" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/compliance" + "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/state" @@ -28,39 +30,52 @@ import ( // Core contains the central business logic for the main consensus' compliance engine. // It is responsible for handling communication for the embedded consensus algorithm. -// NOTE: Core is designed to be non-thread safe and cannot be used in concurrent environment -// user of this object needs to ensure single thread access. +// CAUTION with CONCURRENCY: +// - At the moment, compliance.Core _can not_ process blocks concurrently. Callers of `OnBlockProposal` +// need to ensure single-threaded access. +// - The only exception is calls to `ProcessFinalizedView`, which is the only concurrency-safe +// method of compliance.Core type Core struct { log zerolog.Logger // used to log relevant actions with context config compliance.Config - metrics module.EngineMetrics - tracer module.Tracer - mempool module.MempoolMetrics + engineMetrics module.EngineMetrics + mempoolMetrics module.MempoolMetrics + hotstuffMetrics module.HotstuffMetrics complianceMetrics module.ComplianceMetrics + tracer module.Tracer cleaner storage.Cleaner headers storage.Headers payloads storage.Payloads state protocol.MutableState + // track latest finalized view/height - used to efficiently drop outdated or too-far-ahead blocks + finalizedView counters.StrictMonotonousCounter + finalizedHeight counters.StrictMonotonousCounter pending module.PendingBlockBuffer // pending block cache sync module.BlockRequester hotstuff module.HotStuff + validator hotstuff.Validator voteAggregator hotstuff.VoteAggregator + timeoutAggregator hotstuff.TimeoutAggregator } // NewCore instantiates the business logic for the main consensus' compliance engine. func NewCore( log zerolog.Logger, collector module.EngineMetrics, - tracer module.Tracer, mempool module.MempoolMetrics, + hotstuffMetrics module.HotstuffMetrics, complianceMetrics module.ComplianceMetrics, + tracer module.Tracer, cleaner storage.Cleaner, headers storage.Headers, payloads storage.Payloads, state protocol.MutableState, pending module.PendingBlockBuffer, sync module.BlockRequester, + validator hotstuff.Validator, + hotstuff module.HotStuff, voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, opts ...compliance.Opt, ) (*Core, error) { @@ -69,12 +84,13 @@ func NewCore( apply(&config) } - e := &Core{ + c := &Core{ log: log.With().Str("compliance", "core").Logger(), config: config, - metrics: collector, + engineMetrics: collector, tracer: tracer, - mempool: mempool, + mempoolMetrics: mempool, + hotstuffMetrics: hotstuffMetrics, complianceMetrics: complianceMetrics, cleaner: cleaner, headers: headers, @@ -82,19 +98,33 @@ func NewCore( state: state, pending: pending, sync: sync, - hotstuff: nil, // use `WithConsensus` + hotstuff: hotstuff, + validator: validator, voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, } - e.mempool.MempoolEntries(metrics.ResourceProposal, e.pending.Size()) + // initialize finalized boundary cache + final, err := c.state.Final().Head() + if err != nil { + return nil, fmt.Errorf("could not initialized finalized boundary cache: %w", err) + } + c.ProcessFinalizedBlock(final) + + c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) - return e, nil + return c, nil } // OnBlockProposal handles incoming block proposals. -func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal, inBlockRangeResponse bool) error { +// No errors are expected during normal operation. All returned exceptions +// are potential symptoms of internal state corruption and should be fatal. +func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal) error { block := proposal.Block.ToInternal() header := block.Header + blockID := header.ID() + finalHeight := c.finalizedHeight.Value() + finalView := c.finalizedView.Value() span, _ := c.tracer.StartBlockSpan(context.Background(), header.ID(), trace.CONCompOnBlockProposal) span.SetAttributes( @@ -117,22 +147,40 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc Hex("proposer", header.ProposerID[:]). Hex("parent_signer_indices", header.ParentVoterIndices). Str("traceID", traceID). // traceID is used to connect logs to traces + Uint64("finalized_height", finalHeight). + Uint64("finalized_view", finalView). Logger() log.Info().Msg("block proposal received") + // drop proposals below the finalized threshold + if header.Height <= finalHeight || header.View <= finalView { + log.Debug().Msg("dropping block below finalized boundary") + return nil + } + + // ignore proposals which are too far ahead of our local finalized state + // instead, rely on sync engine to catch up finalization more effectively, and avoid + // large subtree of blocks to be cached. + if header.Height > finalHeight+c.config.SkipNewProposalsThreshold { + log.Debug(). + Uint64("skip_new_proposals_threshold", c.config.SkipNewProposalsThreshold). + Msg("dropping block too far ahead of locally finalized height") + return nil + } + // first, we reject all blocks that we don't need to process: // 1) blocks already in the cache; they will already be processed later // 2) blocks already on disk; they were processed and await finalization // ignore proposals that are already cached - _, cached := c.pending.ByID(header.ID()) + _, cached := c.pending.ByID(blockID) if cached { log.Debug().Msg("skipping already cached proposal") return nil } // ignore proposals that were already processed - _, err := c.headers.ByBlockID(header.ID()) + _, err := c.headers.ByBlockID(blockID) if err == nil { log.Debug().Msg("skipping already processed proposal") return nil @@ -141,20 +189,6 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc return fmt.Errorf("could not check proposal: %w", err) } - // ignore proposals which are too far ahead of our local finalized state - // instead, rely on sync engine to catch up finalization more effectively, and avoid - // large subtree of blocks to be cached. - final, err := c.state.Final().Head() - if err != nil { - return fmt.Errorf("could not get latest finalized header: %w", err) - } - if header.Height > final.Height && header.Height-final.Height > c.config.SkipNewProposalsThreshold { - log.Debug(). - Uint64("final_height", final.Height). - Msg("dropping block too far ahead of locally finalized height") - return nil - } - // there are two possibilities if the proposal is neither already pending // processing in the cache, nor has already been processed: // 1) the proposal is unverifiable because the parent is unknown @@ -169,10 +203,9 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // syncing with range requests for finalized blocks will request for the blocks. _, found := c.pending.ByID(header.ParentID) if found { - // add the block to the cache _ = c.pending.Add(originID, block) - c.mempool.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) + c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) return nil } @@ -180,17 +213,13 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // if the proposal is connected to a block that is neither in the cache, nor // in persistent storage, its direct parent is missing; cache the proposal // and request the parent - _, err = c.headers.ByBlockID(header.ParentID) + parent, err := c.headers.ByBlockID(header.ParentID) if errors.Is(err, storage.ErrNotFound) { - _ = c.pending.Add(originID, block) - - c.mempool.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) - - log.Debug().Msg("requesting missing parent for proposal") + c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) c.sync.RequestBlock(header.ParentID, header.Height-1) - + log.Debug().Msg("requesting missing parent for proposal") return nil } if err != nil { @@ -203,8 +232,8 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // execution of the entire recursion, which might include processing the // proposal's pending children. There is another span within // processBlockProposal that measures the time spent for a single proposal. - err = c.processBlockAndDescendants(block, inBlockRangeResponse) - c.mempool.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) + err = c.processBlockAndDescendants(block, parent) + c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) if err != nil { return fmt.Errorf("could not process block proposal: %w", err) } @@ -222,26 +251,36 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // its pending proposals for its children. By induction, any children connected // to a valid proposal are validly connected to the finalized state and can be // processed as well. -func (c *Core) processBlockAndDescendants(block *flow.Block, inRangeBlockResponse bool) error { - blockID := block.ID() +// No errors are expected during normal operation. All returned exceptions +// are potential symptoms of internal state corruption and should be fatal. +func (c *Core) processBlockAndDescendants(proposal *flow.Block, parent *flow.Header) error { + blockID := proposal.Header.ID() + + log := c.log.With(). + Str("block_id", blockID.String()). + Uint64("block_height", proposal.Header.Height). + Uint64("block_view", proposal.Header.View). + Uint64("parent_view", parent.View). + Logger() // process block itself - err := c.processBlockProposal(block, inRangeBlockResponse) - // child is outdated by the time we started processing it - // => node was probably behind and is catching up. Log as warning - if engine.IsOutdatedInputError(err) { - c.log.Info().Msg("dropped processing of abandoned fork; this might be an indicator that the node is slightly behind") - return nil - } - // the block is invalid; log as error as we desire honest participation - // ToDo: potential slashing - if engine.IsInvalidInputError(err) { - c.log.Warn(). - Err(err). - Msg("received invalid block from other node (potential slashing evidence?)") - return nil - } + err := c.processBlockProposal(proposal, parent) if err != nil { + if checkForAndLogOutdatedInputError(err, log) { + return nil + } + if checkForAndLogInvalidInputError(err, log) { + // notify VoteAggregator about the invalid block + err = c.voteAggregator.InvalidBlock(model.ProposalFromFlow(proposal.Header)) + if err != nil { + if mempool.IsBelowPrunedThresholdError(err) { + log.Warn().Msg("received invalid block, but is below pruned threshold") + return nil + } + return fmt.Errorf("unexpected error notifying vote aggregator about invalid block: %w", err) + } + return nil + } // unexpected error: potentially corrupted internal state => abort processing and escalate error return fmt.Errorf("failed to process block %x: %w", blockID, err) } @@ -254,14 +293,14 @@ func (c *Core) processBlockAndDescendants(block *flow.Block, inRangeBlockRespons return nil } for _, child := range children { - cpr := c.processBlockAndDescendants(child.Message, inRangeBlockResponse) + cpr := c.processBlockAndDescendants(child.Message, proposal.Header) if cpr != nil { // unexpected error: potentially corrupted internal state => abort processing and escalate error return cpr } } - // drop all of the children that should have been processed now + // drop all the children that should have been processed now c.pending.DropForParent(blockID) return nil @@ -269,23 +308,51 @@ func (c *Core) processBlockAndDescendants(block *flow.Block, inRangeBlockRespons // processBlockProposal processes the given block proposal. The proposal must connect to // the finalized state. -func (c *Core) processBlockProposal(block *flow.Block, inRangeBlockResponse bool) error { - header := block.Header - +// Expected errors during normal operations: +// - engine.OutdatedInputError if the block proposal is outdated (e.g. orphaned) +// - engine.InvalidInputError if the block proposal is invalid +func (c *Core) processBlockProposal(proposal *flow.Block, parent *flow.Header) error { startTime := time.Now() - defer c.complianceMetrics.BlockProposalDuration(time.Since(startTime)) + defer func() { + c.hotstuffMetrics.BlockProcessingDuration(time.Since(startTime)) + }() + + header := proposal.Header + blockID := header.ID() - span, ctx := c.tracer.StartBlockSpan(context.Background(), header.ID(), trace.ConCompProcessBlockProposal) + span, ctx := c.tracer.StartBlockSpan(context.Background(), blockID, trace.ConCompProcessBlockProposal) span.SetAttributes( attribute.String("proposer", header.ProposerID.String()), ) defer span.End() + hotstuffProposal := model.ProposalFromFlow(header) + err := c.validator.ValidateProposal(hotstuffProposal) + if err != nil { + if model.IsInvalidBlockError(err) { + return engine.NewInvalidInputErrorf("invalid block proposal: %w", err) + } + if errors.Is(err, model.ErrViewForUnknownEpoch) { + // We have received a proposal, but we don't know the epoch its view is within. + // We know: + // - the parent of this block is valid and was appended to the state (ie. we knew the epoch for it) + // - if we then see this for the child, one of two things must have happened: + // 1. the proposer malicious created the block for a view very far in the future (it's invalid) + // -> in this case we can disregard the block + // 2. no blocks have been finalized within the epoch commitment deadline, and the epoch ended + // (breaking a critical assumption - see EpochCommitSafetyThreshold in protocol.Params for details) + // -> in this case, the network has encountered a critical failure + // - we assume in general that Case 2 will not happen, therefore this must be Case 1 - an invalid block + return engine.NewInvalidInputErrorf("invalid proposal with view from unknown epoch: %w", err) + } + return fmt.Errorf("unexpected error validating proposal: %w", err) + } + log := c.log.With(). Str("chain_id", header.ChainID.String()). Uint64("block_height", header.Height). Uint64("block_view", header.View). - Hex("block_id", logging.Entity(header)). + Hex("block_id", blockID[:]). Hex("parent_id", header.ParentID[:]). Hex("payload_hash", header.PayloadHash[:]). Time("timestamp", header.Timestamp). @@ -295,81 +362,71 @@ func (c *Core) processBlockProposal(block *flow.Block, inRangeBlockResponse bool log.Info().Msg("processing block proposal") // see if the block is a valid extension of the protocol state - err := c.state.Extend(ctx, block) - // if the block proposes an invalid extension of the protocol state, then the block is invalid - if state.IsInvalidExtensionError(err) { - return engine.NewInvalidInputErrorf("invalid extension of protocol state (block: %x, height: %d): %w", - header.ID(), header.Height, err) - } - // protocol state aborted processing of block as it is on an abandoned fork: block is outdated - if state.IsOutdatedExtensionError(err) { - return engine.NewOutdatedInputErrorf("outdated extension of protocol state: %w", err) + block := &flow.Block{ + Header: proposal.Header, + Payload: proposal.Payload, } + err = c.state.Extend(ctx, block) if err != nil { - return fmt.Errorf("could not extend protocol state (block: %x, height: %d): %w", header.ID(), header.Height, err) + if state.IsInvalidExtensionError(err) { + // if the block proposes an invalid extension of the protocol state, then the block is invalid + return engine.NewInvalidInputErrorf("invalid extension of protocol state (block: %x, height: %d): %w", blockID, header.Height, err) + } + if state.IsOutdatedExtensionError(err) { + // protocol state aborted processing of block as it is on an abandoned fork: block is outdated + return engine.NewOutdatedInputErrorf("outdated extension of protocol state: %w", err) + } + // unexpected error: potentially corrupted internal state => abort processing and escalate error + return fmt.Errorf("unexpected exception while extending protocol state with block %x at height %d: %w", blockID, header.Height, err) } - // retrieve the parent - parent, err := c.headers.ByBlockID(header.ParentID) - if err != nil { - return fmt.Errorf("could not retrieve proposal parent: %w", err) - } + // notify vote aggregator about a new block, so that it can start verifying + // votes for it. + c.voteAggregator.AddBlock(hotstuffProposal) // submit the model to hotstuff for processing + // TODO replace with pubsub https://github.com/dapperlabs/flow-go/issues/6395 log.Info().Msg("forwarding block proposal to hotstuff") - - // when the block is in range response, we should wait for hotstuff to finish processing the block, - // otherwise processing the next block might fail because the current block hasn't been added - // to protocol state yet. - if inRangeBlockResponse { - select { - case <-c.hotstuff.SubmitProposal(header, parent.View): - break - case <-c.hotstuff.Done(): - break - } - } else { - c.hotstuff.SubmitProposal(header, parent.View) - } + c.hotstuff.SubmitProposal(hotstuffProposal) return nil } -// OnBlockVote handles incoming block votes. -func (c *Core) OnBlockVote(originID flow.Identifier, vote *messages.BlockVote) error { +// ProcessFinalizedBlock performs pruning of stale data based on finalization event +// removes pending blocks below the finalized view +func (c *Core) ProcessFinalizedBlock(finalized *flow.Header) { + // remove all pending blocks at or below the finalized view + c.pending.PruneByView(finalized.View) + c.finalizedHeight.Set(finalized.Height) + c.finalizedView.Set(finalized.View) - span, _ := c.tracer.StartBlockSpan(context.Background(), vote.BlockID, trace.CONCompOnBlockVote) - span.SetAttributes( - attribute.String("origin_id", originID.String()), - ) - defer span.End() + // always record the metric + c.mempoolMetrics.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) +} - v := &model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, +// checkForAndLogOutdatedInputError checks whether error is an `engine.OutdatedInputError`. +// If this is the case, we emit a log message and return true. +// For any error other than `engine.OutdatedInputError`, this function is a no-op +// and returns false. +func checkForAndLogOutdatedInputError(err error, log zerolog.Logger) bool { + if engine.IsOutdatedInputError(err) { + // child is outdated by the time we started processing it + // => node was probably behind and is catching up. Log as warning + log.Info().Msg("dropped processing of abandoned fork; this might be an indicator that the node is slightly behind") + return true } - - c.log.Info(). - Uint64("block_view", vote.View). - Hex("block_id", vote.BlockID[:]). - Hex("voter", v.SignerID[:]). - Str("vote_id", v.ID().String()). - Msg("block vote received, forwarding block vote to hotstuff vote aggregator") - - // forward the vote to hotstuff for processing - c.voteAggregator.AddVote(v) - - return nil + return false } -// ProcessFinalizedView performs pruning of stale data based on finalization event -// removes pending blocks below the finalized view -func (c *Core) ProcessFinalizedView(finalizedView uint64) { - // remove all pending blocks at or below the finalized view - c.pending.PruneByView(finalizedView) - - // always record the metric - c.mempool.MempoolEntries(metrics.ResourceProposal, c.pending.Size()) +// checkForAndLogInvalidInputError checks whether error is an `engine.InvalidInputError`. +// If this is the case, we emit a log message and return true. +// For any error other than `engine.InvalidInputError`, this function is a no-op +// and returns false. +func checkForAndLogInvalidInputError(err error, log zerolog.Logger) bool { + if engine.IsInvalidInputError(err) { + // the block is invalid; log as error as we desire honest participation + log.Err(err).Msg("received invalid block from other node (potential slashing evidence?)") + return true + } + return false } diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 42b92a232ad..186ab4040b6 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -13,6 +13,7 @@ import ( hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" "github.com/onflow/flow-go/consensus/hotstuff/model" + consensus "github.com/onflow/flow-go/engine/consensus/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" realModule "github.com/onflow/flow-go/module" @@ -24,6 +25,7 @@ import ( netint "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/state" protint "github.com/onflow/flow-go/state/protocol" protocol "github.com/onflow/flow-go/state/protocol/mock" storerr "github.com/onflow/flow-go/storage" @@ -32,10 +34,16 @@ import ( ) func TestComplianceCore(t *testing.T) { - suite.Run(t, new(ComplianceCoreSuite)) + suite.Run(t, new(CoreSuite)) } -type ComplianceCoreSuite struct { +// CoreSuite tests the compliance core logic. +type CoreSuite struct { + CommonSuite +} + +// CommonSuite is shared between compliance core and engine testing. +type CommonSuite struct { suite.Suite // engine parameters @@ -50,33 +58,29 @@ type ComplianceCoreSuite struct { childrenDB map[flow.Identifier][]flow.Slashable[flow.Block] // mocked dependencies - me *module.Local - metrics *metrics.NoopCollector - tracer realModule.Tracer - cleaner *storage.Cleaner - headers *storage.Headers - payloads *storage.Payloads - state *protocol.MutableState - snapshot *protocol.Snapshot - con *mocknetwork.Conduit - net *mocknetwork.Network - prov *mocknetwork.Engine - pending *module.PendingBlockBuffer - hotstuff *module.HotStuff - sync *module.BlockRequester - voteAggregator *hotstuff.VoteAggregator + me *module.Local + metrics *metrics.NoopCollector + tracer realModule.Tracer + cleaner *storage.Cleaner + headers *storage.Headers + payloads *storage.Payloads + state *protocol.MutableState + snapshot *protocol.Snapshot + con *mocknetwork.Conduit + net *mocknetwork.Network + prov *consensus.ProposalProvider + pending *module.PendingBlockBuffer + hotstuff *module.HotStuff + sync *module.BlockRequester + validator *hotstuff.Validator + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator // engine under test core *Core } -func doneChan() <-chan struct{} { - c := make(chan struct{}) - close(c) - return c -} - -func (cs *ComplianceCoreSuite) SetupTest() { +func (cs *CommonSuite) SetupTest() { // seed the RNG rand.Seed(time.Now().UnixNano()) @@ -199,8 +203,8 @@ func (cs *ComplianceCoreSuite) SetupTest() { ) // set up the provider engine - cs.prov = &mocknetwork.Engine{} - cs.prov.On("SubmitLocal", mock.Anything).Return() + cs.prov = &consensus.ProposalProvider{} + cs.prov.On("ProvideProposal", mock.Anything).Return() // set up pending module mock cs.pending = &module.PendingBlockBuffer{} @@ -227,21 +231,17 @@ func (cs *ComplianceCoreSuite) SetupTest() { cs.pending.On("Size").Return(uint(0)) cs.pending.On("PruneByView", mock.Anything).Return() - closed := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - // set up hotstuff module mock - cs.hotstuff = &module.HotStuff{} + cs.hotstuff = module.NewHotStuff(cs.T()) - cs.voteAggregator = &hotstuff.VoteAggregator{} + cs.validator = hotstuff.NewValidator(cs.T()) + cs.voteAggregator = hotstuff.NewVoteAggregator(cs.T()) + cs.timeoutAggregator = hotstuff.NewTimeoutAggregator(cs.T()) // set up synchronization module mock cs.sync = &module.BlockRequester{} cs.sync.On("RequestBlock", mock.Anything, mock.Anything).Return(nil) - cs.sync.On("Done", mock.Anything).Return(closed) + cs.sync.On("Done", mock.Anything).Return(unittest.ClosedChannel()) // set up no-op metrics mock cs.metrics = metrics.NewNoopCollector() @@ -253,25 +253,27 @@ func (cs *ComplianceCoreSuite) SetupTest() { e, err := NewCore( unittest.Logger(), cs.metrics, - cs.tracer, cs.metrics, cs.metrics, + cs.metrics, + cs.tracer, cs.cleaner, cs.headers, cs.payloads, cs.state, cs.pending, cs.sync, + cs.validator, + cs.hotstuff, cs.voteAggregator, + cs.timeoutAggregator, ) require.NoError(cs.T(), err, "engine initialization should pass") cs.core = e - // assign engine with consensus & synchronization - cs.core.hotstuff = cs.hotstuff } -func (cs *ComplianceCoreSuite) TestOnBlockProposalValidParent() { +func (cs *CoreSuite) TestOnBlockProposalValidParent() { // create a proposal that directly descends from the latest finalized header originID := cs.participants[1].NodeID @@ -281,20 +283,20 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalValidParent() { // store the data for retrieval cs.headerDB[block.Header.ParentID] = cs.head - cs.hotstuff.On("SubmitProposal", block.Header, cs.head.View).Return(doneChan()) + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal, false) + err := cs.core.OnBlockProposal(originID, proposal) require.NoError(cs.T(), err, "valid block proposal should pass") // we should extend the state with the header cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) - - // we should submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalValidAncestor() { +func (cs *CoreSuite) TestOnBlockProposalValidAncestor() { // create a proposal that has two ancestors in the cache originID := cs.participants[1].NodeID @@ -307,20 +309,20 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalValidAncestor() { cs.headerDB[parent.ID()] = parent.Header cs.headerDB[ancestor.ID()] = ancestor.Header - cs.hotstuff.On("SubmitProposal", block.Header, parent.Header.View).Return(doneChan()) + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal) // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal, false) + err := cs.core.OnBlockProposal(originID, proposal) require.NoError(cs.T(), err, "valid block proposal should pass") // we should extend the state with the header cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) - - // we should submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalSkipProposalThreshold() { +func (cs *CoreSuite) TestOnBlockProposalSkipProposalThreshold() { // create a proposal which is far enough ahead to be dropped originID := cs.participants[1].NodeID @@ -328,15 +330,21 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalSkipProposalThreshold() { block.Header.Height = cs.head.Height + compliance.DefaultConfig().SkipNewProposalsThreshold + 1 proposal := unittest.ProposalFromBlock(&block) - err := cs.core.OnBlockProposal(originID, proposal, false) + err := cs.core.OnBlockProposal(originID, proposal) require.NoError(cs.T(), err) // block should be dropped - not added to state or cache cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything) + cs.validator.AssertNotCalled(cs.T(), "ValidateProposal", mock.Anything) cs.pending.AssertNotCalled(cs.T(), "Add", originID, mock.Anything) } -func (cs *ComplianceCoreSuite) TestOnBlockProposalInvalidExtension() { +// TestOnBlockProposal_FailsHotStuffValidation tests that a proposal which fails HotStuff validation. +// - should not go through protocol state validation +// - should not be added to the state +// - we should not attempt to process its children +// - we should notify VoteAggregator, for known errors +func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() { // create a proposal that has two ancestors in the cache originID := cs.participants[1].NodeID @@ -344,32 +352,142 @@ func (cs *ComplianceCoreSuite) TestOnBlockProposalInvalidExtension() { parent := unittest.BlockWithParentFixture(ancestor.Header) block := unittest.BlockWithParentFixture(parent.Header) proposal := unittest.ProposalFromBlock(block) + hotstuffProposal := model.ProposalFromFlow(block.Header) // store the data for retrieval cs.headerDB[parent.ID()] = parent.Header cs.headerDB[ancestor.ID()] = ancestor.Header - // make sure we fail to extend the state - *cs.state = protocol.MutableState{} - cs.state.On("Final").Return( - func() protint.Snapshot { - return cs.snapshot - }, - ) - cs.state.On("Extend", mock.Anything, mock.Anything).Return(errors.New("dummy error")) + cs.Run("invalid block error", func() { + // the block fails HotStuff validation + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.InvalidBlockError{}) + // we should notify VoteAggregator about the invalid block + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("view for unknown epoch error", func() { + // the block fails HotStuff validation + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.ErrViewForUnknownEpoch) + + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("unexpected error", func() { + // the block fails HotStuff validation + unexpectedErr := errors.New("generic unexpected error") + *cs.validator = *hotstuff.NewValidator(cs.T()) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(unexpectedErr) + + // the error should be propagated + err := cs.core.OnBlockProposal(originID, proposal) + require.ErrorIs(cs.T(), err, unexpectedErr) + + // we should not extend the state with the header + cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) +} - // it should be processed without error - err := cs.core.OnBlockProposal(originID, proposal, false) - require.Error(cs.T(), err, "proposal with invalid extension should fail") +// TestOnBlockProposal_FailsProtocolStateValidation tests processing a proposal which passes HotStuff validation, +// but fails protocol state validation +// - should not be added to the state +// - we should not attempt to process its children +// - we should notify VoteAggregator, for known errors +func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() { - // we should extend the state with the header - cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) + // create a proposal that has two ancestors in the cache + originID := cs.participants[1].NodeID + ancestor := unittest.BlockWithParentFixture(cs.head) + parent := unittest.BlockWithParentFixture(ancestor.Header) + block := unittest.BlockWithParentFixture(parent.Header) + proposal := unittest.ProposalFromBlock(block) + hotstuffProposal := model.ProposalFromFlow(block.Header) - // we should not submit the proposal to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) + // store the data for retrieval + cs.headerDB[parent.ID()] = parent.Header + cs.headerDB[ancestor.ID()] = ancestor.Header + + // the block passes HotStuff validation + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + + cs.Run("invalid block", func() { + // make sure we fail to extend the state + *cs.state = protocol.MutableState{} + cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) + cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewInvalidExtensionError("")) + // we should notify VoteAggregator about the invalid block + cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil) + + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("outdated block", func() { + // make sure we fail to extend the state + *cs.state = protocol.MutableState{} + cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) + cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewOutdatedExtensionError("")) + + // the expected error should be handled within the Core + err := cs.core.OnBlockProposal(originID, proposal) + require.NoError(cs.T(), err, "proposal with invalid extension should fail") + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) + + cs.Run("unexpected error", func() { + // make sure we fail to extend the state + *cs.state = protocol.MutableState{} + cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot }) + unexpectedErr := errors.New("unexpected generic error") + cs.state.On("Extend", mock.Anything, mock.Anything).Return(unexpectedErr) + + // it should be processed without error + err := cs.core.OnBlockProposal(originID, proposal) + require.ErrorIs(cs.T(), err, unexpectedErr) + + // we should extend the state with the header + cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block) + // we should not pass the block to hotstuff + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything) + // we should not attempt to process the children + cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything) + }) } -func (cs *ComplianceCoreSuite) TestProcessBlockAndDescendants() { +func (cs *CoreSuite) TestProcessBlockAndDescendants() { // create three children blocks parent := unittest.BlockWithParentFixture(cs.head) @@ -391,56 +509,31 @@ func (cs *ComplianceCoreSuite) TestProcessBlockAndDescendants() { cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending2) cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending3) - cs.hotstuff.On("SubmitProposal", parent.Header, cs.head.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block1.Header, parent.Header.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block2.Header, parent.Header.View).Return(doneChan()).Once() - cs.hotstuff.On("SubmitProposal", block3.Header, parent.Header.View).Return(doneChan()).Once() + for _, block := range []*flow.Block{parent, block1, block2, block3} { + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil) + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once() + } // execute the connected children handling - err := cs.core.processBlockAndDescendants(parent, false) + err := cs.core.processBlockAndDescendants(parent, cs.head) require.NoError(cs.T(), err, "should pass handling children") - // check that we submitted each child to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) - // make sure we drop the cache after trying to process cs.pending.AssertCalled(cs.T(), "DropForParent", parent.Header.ID()) } -func (cs *ComplianceCoreSuite) TestOnSubmitVote() { - // create a vote - originID := unittest.IdentifierFixture() - vote := messages.BlockVote{ - BlockID: unittest.IdentifierFixture(), - View: rand.Uint64(), - SigData: unittest.SignatureFixture(), - } - - cs.voteAggregator.On("AddVote", &model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, - }).Return() - - // execute the vote submission - err := cs.core.OnBlockVote(originID, &vote) - require.NoError(cs.T(), err, "block vote should pass") - - // check that submit vote was called with correct parameters - cs.hotstuff.AssertExpectations(cs.T()) -} - -func (cs *ComplianceCoreSuite) TestProposalBufferingOrder() { +func (cs *CoreSuite) TestProposalBufferingOrder() { // create a proposal that we will not submit until the end originID := cs.participants[1].NodeID - block := unittest.BlockWithParentFixture(cs.head) - missing := unittest.ProposalFromBlock(block) + missingBlock := unittest.BlockWithParentFixture(cs.head) + missingProposal := unittest.ProposalFromBlock(missingBlock) // create a chain of descendants var proposals []*messages.BlockProposal - parent := missing + parent := missingProposal for i := 0; i < 3; i++ { descendant := unittest.BlockWithParentFixture(&parent.Block.Header) proposal := unittest.ProposalFromBlock(descendant) @@ -451,47 +544,51 @@ func (cs *ComplianceCoreSuite) TestProposalBufferingOrder() { // replace the engine buffer with the real one cs.core.pending = real.NewPendingBlocks() - // process all of the descendants - for _, proposal := range proposals { - - // check that we request the ancestor block each time - cs.sync.On("RequestBlock", mock.Anything, mock.Anything).Once().Run( - func(args mock.Arguments) { - ancestorID := args.Get(0).(flow.Identifier) - assert.Equal(cs.T(), missing.Block.Header.ID(), ancestorID, "should always request root block") - }, - ) + // check that we request the ancestor block each time + cs.sync.On("RequestBlock", missingBlock.Header.ID(), missingBlock.Header.Height).Times(len(proposals)) + // process all the descendants + for _, proposal := range proposals { // process and make sure no error occurs (as they are unverifiable) - err := cs.core.OnBlockProposal(originID, proposal, false) + err := cs.core.OnBlockProposal(originID, proposal) require.NoError(cs.T(), err, "proposal buffering should pass") // make sure no block is forwarded to hotstuff - cs.hotstuff.AssertExpectations(cs.T()) + cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", model.ProposalFromFlow(&proposal.Block.Header)) } - // check that we submit ech proposal in order - *cs.hotstuff = module.HotStuff{} - index := 0 - order := []flow.Identifier{ - missing.Block.Header.ID(), - proposals[0].Block.Header.ID(), - proposals[1].Block.Header.ID(), - proposals[2].Block.Header.ID(), + // check that we submit each proposal in a valid order + // - we must process the missingProposal parent first + // - we can process the children next, in any order + cs.validator.On("ValidateProposal", mock.Anything).Return(nil).Times(4) + + calls := 0 // track # of calls to SubmitProposal + unprocessed := map[flow.Identifier]struct{}{ // track un-processed proposals + missingProposal.Block.Header.ID(): {}, + proposals[0].Block.Header.ID(): {}, + proposals[1].Block.Header.ID(): {}, + proposals[2].Block.Header.ID(): {}, } - cs.hotstuff.On("SubmitProposal", mock.Anything, mock.Anything).Times(4).Run( + cs.hotstuff.On("SubmitProposal", mock.Anything).Times(4).Run( func(args mock.Arguments) { - header := args.Get(0).(*flow.Header) - assert.Equal(cs.T(), order[index], header.ID(), "should submit correct header to hotstuff") - index++ - cs.headerDB[header.ID()] = header + proposal := args.Get(0).(*model.Proposal) + header := proposal.Block + if calls == 0 { + // first header processed must be the common parent + assert.Equal(cs.T(), missingProposal.Block.Header.ID(), header.BlockID) + } + // mark the proposal as processed + delete(unprocessed, header.BlockID) + cs.headerDB[header.BlockID] = model.ProposalToFlow(proposal) + calls++ }, - ).Return(doneChan()) + ) + cs.voteAggregator.On("AddBlock", mock.Anything).Times(4) // process the root proposal - err := cs.core.OnBlockProposal(originID, missing, false) + err := cs.core.OnBlockProposal(originID, missingProposal) require.NoError(cs.T(), err, "root proposal should pass") - // make sure we submitted all four proposals - cs.hotstuff.AssertExpectations(cs.T()) + // all proposals should be processed + assert.Len(cs.T(), unprocessed, 0) } diff --git a/engine/consensus/compliance/engine.go b/engine/consensus/compliance/engine.go index 7e724f0f553..b0af778c7da 100644 --- a/engine/consensus/compliance/engine.go +++ b/engine/consensus/compliance/engine.go @@ -1,502 +1,190 @@ package compliance import ( - "context" - "errors" "fmt" - "time" "github.com/rs/zerolog" "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/tracker" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/common/fifoqueue" - "github.com/onflow/flow-go/engine/consensus/sealing/counters" - "github.com/onflow/flow-go/model/events" + "github.com/onflow/flow-go/engine/consensus" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/lifecycle" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/logging" ) -// defaultRangeResponseQueueCapacity maximum capacity of block range responses queue -const defaultRangeResponseQueueCapacity = 100 +// defaultBlockQueueCapacity maximum capacity of inbound queue for `messages.BlockProposal`s +const defaultBlockQueueCapacity = 10_000 -// defaultBlockQueueCapacity maximum capacity of block proposals queue -const defaultBlockQueueCapacity = 10000 - -// defaultVoteQueueCapacity maximum capacity of block votes queue -const defaultVoteQueueCapacity = 1000 - -// Engine is a wrapper struct for `Core` which implements consensus algorithm. -// Engine is responsible for handling incoming messages, queueing for processing, broadcasting proposals. +// Engine is a wrapper around `compliance.Core`. The Engine queues inbound messages, relevant +// node-internal notifications, and manages the worker routines processing the inbound events, +// and forwards outbound messages to the networking layer. +// `compliance.Core` implements the actual compliance logic. +// Implements consensus.Compliance interface. type Engine struct { - unit *engine.Unit - lm *lifecycle.LifecycleManager - log zerolog.Logger - mempool module.MempoolMetrics - metrics module.EngineMetrics - me module.Local - headers storage.Headers - payloads storage.Payloads - tracer module.Tracer - state protocol.State - prov network.Engine - core *Core - pendingBlocks engine.MessageStore - pendingRangeResponses engine.MessageStore - pendingVotes engine.MessageStore - messageHandler *engine.MessageHandler - finalizedView counters.StrictMonotonousCounter - finalizationEventsNotifier engine.Notifier - con network.Conduit - stopHotstuff context.CancelFunc -} + *component.ComponentManager + log zerolog.Logger + mempoolMetrics module.MempoolMetrics + engineMetrics module.EngineMetrics + me module.Local + headers storage.Headers + payloads storage.Payloads + tracer module.Tracer + state protocol.State + core *Core + pendingBlocks *fifoqueue.FifoQueue // queue for processing inbound blocks + pendingBlocksNotifier engine.Notifier + finalizedBlockTracker *tracker.NewestBlockTracker + finalizedBlockNotifier engine.Notifier +} + +var _ consensus.Compliance = (*Engine)(nil) func NewEngine( log zerolog.Logger, - net network.Network, me module.Local, - prov network.Engine, - core *Core) (*Engine, error) { - - rangeResponseQueue, err := fifoqueue.NewFifoQueue( - defaultRangeResponseQueueCapacity, - fifoqueue.WithLengthObserver(func(len int) { core.mempool.MempoolEntries(metrics.ResourceBlockResponseQueue, uint(len)) }), - ) - - if err != nil { - return nil, fmt.Errorf("failed to create queue for block responses: %w", err) - } + core *Core, +) (*Engine, error) { - pendingRangeResponses := &engine.FifoMessageStore{ - FifoQueue: rangeResponseQueue, - } - - // FIFO queue for block proposals + // Inbound FIFO queue for `messages.BlockProposal`s blocksQueue, err := fifoqueue.NewFifoQueue( defaultBlockQueueCapacity, - fifoqueue.WithLengthObserver(func(len int) { core.mempool.MempoolEntries(metrics.ResourceBlockProposalQueue, uint(len)) }), + fifoqueue.WithLengthObserver(func(len int) { core.mempoolMetrics.MempoolEntries(metrics.ResourceBlockProposalQueue, uint(len)) }), ) if err != nil { - return nil, fmt.Errorf("failed to create queue for inbound receipts: %w", err) - } - - pendingBlocks := &engine.FifoMessageStore{ - FifoQueue: blocksQueue, + return nil, fmt.Errorf("failed to create queue for inbound block proposals: %w", err) } - // FIFO queue for block votes - votesQueue, err := fifoqueue.NewFifoQueue( - defaultVoteQueueCapacity, - fifoqueue.WithLengthObserver(func(len int) { core.mempool.MempoolEntries(metrics.ResourceBlockVoteQueue, uint(len)) }), - ) - if err != nil { - return nil, fmt.Errorf("failed to create queue for inbound approvals: %w", err) - } - pendingVotes := &engine.FifoMessageStore{FifoQueue: votesQueue} - - // define message queueing behaviour - handler := engine.NewMessageHandler( - log.With().Str("compliance", "engine").Logger(), - engine.NewNotifier(), - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*messages.BlockResponse) - if ok { - core.metrics.MessageReceived(metrics.EngineCompliance, metrics.MessageBlockResponse) - } - return ok - }, - Store: pendingRangeResponses, - }, - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*messages.BlockProposal) - if ok { - core.metrics.MessageReceived(metrics.EngineCompliance, metrics.MessageBlockProposal) - } - return ok - }, - Store: pendingBlocks, - }, - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*events.SyncedBlock) - if ok { - core.metrics.MessageReceived(metrics.EngineCompliance, metrics.MessageSyncedBlock) - } - return ok - }, - Map: func(msg *engine.Message) (*engine.Message, bool) { - syncedBlock := msg.Payload.(*events.SyncedBlock) - msg = &engine.Message{ - OriginID: msg.OriginID, - Payload: &messages.BlockProposal{ - Block: syncedBlock.Block, - }, - } - return msg, true - }, - Store: pendingBlocks, - }, - engine.Pattern{ - Match: func(msg *engine.Message) bool { - _, ok := msg.Payload.(*messages.BlockVote) - if ok { - core.metrics.MessageReceived(metrics.EngineCompliance, metrics.MessageBlockVote) - } - return ok - }, - Store: pendingVotes, - }, - ) - eng := &Engine{ - unit: engine.NewUnit(), - lm: lifecycle.NewLifecycleManager(), - log: log.With().Str("compliance", "engine").Logger(), - me: me, - mempool: core.mempool, - metrics: core.metrics, - headers: core.headers, - payloads: core.payloads, - pendingRangeResponses: pendingRangeResponses, - pendingBlocks: pendingBlocks, - pendingVotes: pendingVotes, - state: core.state, - tracer: core.tracer, - prov: prov, - core: core, - messageHandler: handler, - finalizationEventsNotifier: engine.NewNotifier(), - } - - // register the core with the network layer and store the conduit - eng.con, err = net.Register(channels.ConsensusCommittee, eng) - if err != nil { - return nil, fmt.Errorf("could not register core: %w", err) - } + log: log.With().Str("compliance", "engine").Logger(), + me: me, + mempoolMetrics: core.mempoolMetrics, + engineMetrics: core.engineMetrics, + headers: core.headers, + payloads: core.payloads, + pendingBlocks: blocksQueue, + state: core.state, + tracer: core.tracer, + core: core, + pendingBlocksNotifier: engine.NewNotifier(), + finalizedBlockTracker: tracker.NewNewestBlockTracker(), + finalizedBlockNotifier: engine.NewNotifier(), + } + + // create the component manager and worker threads + eng.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(eng.processBlocksLoop). + AddWorker(eng.finalizationProcessingLoop). + Build() return eng, nil } -// WithConsensus adds the consensus algorithm to the engine. This must be -// called before the engine can start. -func (e *Engine) WithConsensus(hot module.HotStuff) *Engine { - e.core.hotstuff = hot - return e -} - -// Ready returns a ready channel that is closed once the engine has fully -// started. For consensus engine, this is true once the underlying consensus -// algorithm has started. -func (e *Engine) Ready() <-chan struct{} { - if e.core.hotstuff == nil { - panic("must initialize compliance engine with hotstuff engine") - } - e.lm.OnStart(func() { - e.unit.Launch(e.loop) - e.unit.Launch(e.finalizationProcessingLoop) - - ctx, cancel := context.WithCancel(context.Background()) - signalerCtx, hotstuffErrChan := irrecoverable.WithSignaler(ctx) - e.stopHotstuff = cancel +// processBlocksLoop processes available block, vote, and timeout messages as they are queued. +func (e *Engine) processBlocksLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() - // TODO: this workaround for handling fatal HotStuff errors is required only - // because this engine and epochmgr do not use the Component pattern yet - e.unit.Launch(func() { - e.handleHotStuffError(hotstuffErrChan) - }) - - e.core.hotstuff.Start(signalerCtx) - // wait for request handler to startup - - <-e.core.hotstuff.Ready() - }) - return e.lm.Started() -} - -// Done returns a done channel that is closed once the engine has fully stopped. -// For the consensus engine, we wait for hotstuff to finish. -func (e *Engine) Done() <-chan struct{} { - e.lm.OnStop(func() { - e.log.Info().Msg("shutting down hotstuff eventloop") - e.stopHotstuff() - <-e.core.hotstuff.Done() - e.log.Info().Msg("all components have been shut down") - <-e.unit.Done() - }) - return e.lm.Stopped() -} - -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - err := e.ProcessLocal(event) - if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// Submit submits the given event from the node with the given origin ID -// for processing in a non-blocking manner. It returns instantly and logs -// a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { - err := e.Process(channel, originID, event) - if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing event") - } -} - -// ProcessLocal processes an event originating on the local node. -func (e *Engine) ProcessLocal(event interface{}) error { - return e.messageHandler.Process(e.me.NodeID(), event) -} - -// Process processes the given event from the node with the given origin ID in -// a blocking manner. It returns the potential processing error when done. -func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - err := e.messageHandler.Process(originID, event) - if err != nil { - if engine.IsIncompatibleInputTypeError(err) { - e.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, event, channel) - return nil - } - return fmt.Errorf("unexpected error while processing engine message: %w", err) - } - return nil -} - -func (e *Engine) loop() { + doneSignal := ctx.Done() + newMessageSignal := e.pendingBlocksNotifier.Channel() for { select { - case <-e.unit.Quit(): + case <-doneSignal: return - case <-e.messageHandler.GetNotifier(): - err := e.processAvailableMessages() + case <-newMessageSignal: + err := e.processQueuedBlocks(doneSignal) // no errors expected during normal operations if err != nil { - e.log.Fatal().Err(err).Msg("internal error processing queued message") + ctx.Throw(err) } } } } -func (e *Engine) processAvailableMessages() error { - +// processQueuedBlocks processes any available messages until the message queue is empty. +// Only returns when all inbound queues are empty (or the engine is terminated). +// No errors are expected during normal operation. All returned exceptions are potential +// symptoms of internal state corruption and should be fatal. +func (e *Engine) processQueuedBlocks(doneSignal <-chan struct{}) error { for { - // TODO prioritization - // eg: msg := engine.SelectNextMessage() - msg, ok := e.pendingRangeResponses.Get() - if ok { - blockResponse := msg.Payload.(*messages.BlockResponse) - for _, block := range blockResponse.Blocks { - // process each block and indicate it's from a range of blocks - err := e.core.OnBlockProposal(msg.OriginID, &messages.BlockProposal{ - Block: block, - }, true) - - if err != nil { - return fmt.Errorf("could not handle block proposal: %w", err) - } - } - continue + select { + case <-doneSignal: + return nil + default: } - msg, ok = e.pendingBlocks.Get() + msg, ok := e.pendingBlocks.Pop() if ok { - err := e.core.OnBlockProposal(msg.OriginID, msg.Payload.(*messages.BlockProposal), true) + inBlock := msg.(flow.Slashable[messages.BlockProposal]) + err := e.core.OnBlockProposal(inBlock.OriginID, inBlock.Message) + e.core.engineMetrics.MessageHandled(metrics.EngineCompliance, metrics.MessageBlockProposal) if err != nil { return fmt.Errorf("could not handle block proposal: %w", err) } continue } - msg, ok = e.pendingVotes.Get() - if ok { - err := e.core.OnBlockVote(msg.OriginID, msg.Payload.(*messages.BlockVote)) - if err != nil { - return fmt.Errorf("could not handle block vote: %w", err) - } - continue - } - - // when there is no more messages in the queue, back to the loop to wait + // when there are no more messages in the queue, back to the processBlocksLoop to wait // for the next incoming message to arrive. return nil } } -// SendVote will send a vote to the desired node. -func (e *Engine) SendVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) error { - - log := e.log.With(). - Hex("block_id", blockID[:]). - Uint64("block_view", view). - Hex("recipient_id", recipientID[:]). - Logger() - - log.Info().Msg("processing vote transmission request from hotstuff") - - // build the vote message - vote := &messages.BlockVote{ - BlockID: blockID, - View: view, - SigData: sigData, - } - - // TODO: this is a hot-fix to mitigate the effects of the following Unicast call blocking occasionally - e.unit.Launch(func() { - // send the vote the desired recipient - err := e.con.Unicast(vote, recipientID) - if err != nil { - log.Warn().Err(err).Msg("could not send vote") - return - } - e.metrics.MessageSent(metrics.EngineCompliance, metrics.MessageBlockVote) - log.Info().Msg("block vote transmitted") - }) - - return nil -} - -// BroadcastProposalWithDelay will propagate a block proposal to all non-local consensus nodes. -// Note the header has incomplete fields, because it was converted from a hotstuff. -func (e *Engine) BroadcastProposalWithDelay(header *flow.Header, delay time.Duration) error { - - // first, check that we are the proposer of the block - if header.ProposerID != e.me.NodeID() { - return fmt.Errorf("cannot broadcast proposal with non-local proposer (%x)", header.ProposerID) - } - - // get the parent of the block - parent, err := e.headers.ByBlockID(header.ParentID) - if err != nil { - return fmt.Errorf("could not retrieve proposal parent: %w", err) - } - - // fill in the fields that can't be populated by HotStuff - header.ChainID = parent.ChainID - header.Height = parent.Height + 1 - - // retrieve the payload for the block - payload, err := e.payloads.ByBlockID(header.ID()) - if err != nil { - return fmt.Errorf("could not retrieve payload for proposal: %w", err) - } - - log := e.log.With(). - Str("chain_id", header.ChainID.String()). - Uint64("block_height", header.Height). - Uint64("block_view", header.View). - Hex("block_id", logging.Entity(header)). - Hex("parent_id", header.ParentID[:]). - Hex("payload_hash", header.PayloadHash[:]). - Int("gaurantees_count", len(payload.Guarantees)). - Int("seals_count", len(payload.Seals)). - Int("receipts_count", len(payload.Receipts)). - Time("timestamp", header.Timestamp). - Hex("signers", header.ParentVoterIndices). - Dur("delay", delay). - Logger() - - log.Debug().Msg("processing proposal broadcast request from hotstuff") - - // retrieve all consensus nodes without our ID - recipients, err := e.state.AtBlockID(header.ParentID).Identities(filter.And( - filter.HasRole(flow.RoleConsensus), - filter.Not(filter.HasNodeID(e.me.NodeID())), - )) - if err != nil { - return fmt.Errorf("could not get consensus recipients: %w", err) - } - - e.unit.LaunchAfter(delay, func() { - - go e.core.hotstuff.SubmitProposal(header, parent.View) - - // NOTE: some fields are not needed for the message - // - proposer ID is conveyed over the network message - // - the payload hash is deduced from the payload - block := &flow.Block{ - Header: header, - Payload: payload, - } - proposal := messages.NewBlockProposal(block) - - // broadcast the proposal to consensus nodes - err = e.con.Publish(proposal, recipients.NodeIDs()...) - if errors.Is(err, network.EmptyTargetList) { - return - } - if err != nil { - log.Error().Err(err).Msg("could not send proposal message") - } - - e.metrics.MessageSent(metrics.EngineCompliance, metrics.MessageBlockProposal) - - log.Info().Msg("block proposal broadcasted") - - // submit the proposal to the provider engine to forward it to other - // node roles - e.prov.SubmitLocal(proposal) - }) - - return nil -} - -// BroadcastProposal will propagate a block proposal to all non-local consensus nodes. -// Note the header has incomplete fields, because it was converted from a hotstuff. -func (e *Engine) BroadcastProposal(header *flow.Header) error { - return e.BroadcastProposalWithDelay(header, 0) -} - // OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// (1) Informs sealing.Core about finalization of respective block. +// It informs compliance.Core about finalization of the respective block. // // CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages // from external nodes cannot be considered as inputs to this function func (e *Engine) OnFinalizedBlock(block *model.Block) { - if e.finalizedView.Set(block.View) { - e.finalizationEventsNotifier.Notify() + if e.finalizedBlockTracker.Track(block) { + e.finalizedBlockNotifier.Notify() } } -// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events -func (e *Engine) finalizationProcessingLoop() { - finalizationNotifier := e.finalizationEventsNotifier.Channel() - for { - select { - case <-e.unit.Quit(): - return - case <-finalizationNotifier: - e.core.ProcessFinalizedView(e.finalizedView.Value()) - } +// OnBlockProposal feeds a new block proposal into the processing pipeline. +// Incoming proposals are queued and eventually dispatched by worker. +func (e *Engine) OnBlockProposal(proposal flow.Slashable[messages.BlockProposal]) { + e.core.engineMetrics.MessageReceived(metrics.EngineCompliance, metrics.MessageBlockProposal) + if e.pendingBlocks.Push(proposal) { + e.pendingBlocksNotifier.Notify() + } else { + e.core.engineMetrics.InboundMessageDropped(metrics.EngineCompliance, metrics.MessageBlockProposal) } } -// handleHotStuffError accepts the error channel from the HotStuff component and -// crashes the node if any error is detected. -// -// TODO: this function should be removed in favour of refactoring this engine and -// the epochmgr engine to use the Component pattern, so that irrecoverable errors -// can be bubbled all the way to the node scaffold -func (e *Engine) handleHotStuffError(hotstuffErrs <-chan error) { +// OnSyncedBlock feeds a block obtained from sync proposal into the processing pipeline. +// Incoming proposals are queued and eventually dispatched by worker. +func (e *Engine) OnSyncedBlock(syncedBlock flow.Slashable[messages.BlockProposal]) { + e.core.engineMetrics.MessageReceived(metrics.EngineCompliance, metrics.MessageSyncedBlock) + if e.pendingBlocks.Push(syncedBlock) { + e.pendingBlocksNotifier.Notify() + } else { + e.core.engineMetrics.InboundMessageDropped(metrics.EngineCompliance, metrics.MessageSyncedBlock) + } +} + +// finalizationProcessingLoop is a separate goroutine that performs processing of finalization events +func (e *Engine) finalizationProcessingLoop(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + doneSignal := ctx.Done() + blockFinalizedSignal := e.finalizedBlockNotifier.Channel() for { select { - case <-e.unit.Quit(): + case <-doneSignal: return - case err := <-hotstuffErrs: - if err != nil { - e.log.Fatal().Err(err).Msg("encountered fatal error in HotStuff") + case <-blockFinalizedSignal: + // retrieve the latest finalized header, so we know the height + finalHeader, err := e.headers.ByBlockID(e.finalizedBlockTracker.NewestBlock().BlockID) + if err != nil { // no expected errors + ctx.Throw(err) } + e.core.ProcessFinalizedBlock(finalHeader) } } } diff --git a/engine/consensus/compliance/engine_test.go b/engine/consensus/compliance/engine_test.go index 9f0a2b6b93b..a6a0d7c4122 100644 --- a/engine/consensus/compliance/engine_test.go +++ b/engine/consensus/compliance/engine_test.go @@ -1,232 +1,133 @@ package compliance import ( - "math/rand" + "context" "sync" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/consensus/hotstuff/model" - "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/irrecoverable" modulemock "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/utils/unittest" ) func TestComplianceEngine(t *testing.T) { - suite.Run(t, new(ComplianceSuite)) + suite.Run(t, new(EngineSuite)) } -type ComplianceSuite struct { - ComplianceCoreSuite +// EngineSuite tests the compliance engine. +type EngineSuite struct { + CommonSuite + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error engine *Engine } -func (cs *ComplianceSuite) SetupTest() { - cs.ComplianceCoreSuite.SetupTest() - e, err := NewEngine(unittest.Logger(), cs.net, cs.me, cs.prov, cs.core) +func (cs *EngineSuite) SetupTest() { + cs.CommonSuite.SetupTest() + + e, err := NewEngine(unittest.Logger(), cs.me, cs.core) require.NoError(cs.T(), err) cs.engine = e - ready := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - cs.hotstuff.On("Start", mock.Anything) - cs.hotstuff.On("Ready", mock.Anything).Return(ready) - <-cs.engine.Ready() -} - -// TestSendVote tests that single vote can be send and properly processed -func (cs *ComplianceSuite) TestSendVote() { - // create parameters to send a vote - blockID := unittest.IdentifierFixture() - view := rand.Uint64() - sig := unittest.SignatureFixture() - recipientID := unittest.IdentifierFixture() - - // submit the vote - err := cs.engine.SendVote(blockID, view, sig, recipientID) - require.NoError(cs.T(), err, "should pass send vote") - - done := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - cs.hotstuff.On("Done", mock.Anything).Return(done) + cs.ctx, cs.cancel, cs.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + cs.engine.Start(cs.ctx) + go unittest.FailOnIrrecoverableError(cs.T(), cs.ctx.Done(), cs.errs) - // The vote is transmitted asynchronously. We allow 10ms for the vote to be received: - <-time.After(10 * time.Millisecond) - <-cs.engine.Done() - - // check it was called with right params - vote := messages.BlockVote{ - BlockID: blockID, - View: view, - SigData: sig, - } - cs.con.AssertCalled(cs.T(), "Unicast", &vote, recipientID) + unittest.AssertClosesBefore(cs.T(), cs.engine.Ready(), time.Second) } -// TestBroadcastProposalWithDelay tests broadcasting proposals with different -// inputs -func (cs *ComplianceSuite) TestBroadcastProposalWithDelay() { - - // add execution node to participants to make sure we exclude them from broadcast - cs.participants = append(cs.participants, unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution))) - - // generate a parent with height and chain ID set - parent := unittest.BlockHeaderFixture() - parent.ChainID = "test" - parent.Height = 10 - cs.headerDB[parent.ID()] = parent - - // create a block with the parent and store the payload with correct ID - block := unittest.BlockWithParentFixture(parent) - block.Header.ProposerID = cs.myID - cs.payloadDB[block.ID()] = block.Payload - - // keep a duplicate of the correct header to check against leader - header := block.Header - - // unset chain and height to make sure they are correctly reconstructed - block.Header.ChainID = "" - block.Header.Height = 0 - - cs.hotstuff.On("SubmitProposal", block.Header, parent.View).Return(doneChan()).Once() - - // submit to broadcast proposal - err := cs.engine.BroadcastProposalWithDelay(block.Header, 0) - require.NoError(cs.T(), err, "header broadcast should pass") - - // make sure chain ID and height were reconstructed and - // we broadcast to correct nodes - header.ChainID = "test" - header.Height = 11 - msg := &messages.BlockProposal{ - Block: messages.UntrustedBlockFromInternal(block), +// TearDownTest stops the engine and checks there are no errors thrown to the SignallerContext. +func (cs *EngineSuite) TearDownTest() { + cs.cancel() + unittest.RequireCloseBefore(cs.T(), cs.engine.Done(), time.Second, "engine failed to stop") + select { + case err := <-cs.errs: + assert.NoError(cs.T(), err) + default: } - - done := func() <-chan struct{} { - channel := make(chan struct{}) - close(channel) - return channel - }() - - cs.hotstuff.On("Done", mock.Anything).Return(done) - - <-time.After(10 * time.Millisecond) - <-cs.engine.Done() - cs.con.AssertCalled(cs.T(), "Publish", msg, cs.participants[1].NodeID, cs.participants[2].NodeID) - - // should fail with wrong proposer - header.ProposerID = unittest.IdentifierFixture() - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with wrong proposer") - header.ProposerID = cs.myID - - // should fail with changed (missing) parent - header.ParentID[0]++ - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with missing parent") - header.ParentID[0]-- - - // should fail with wrong block ID (payload unavailable) - header.View++ - err = cs.engine.BroadcastProposalWithDelay(header, 0) - require.Error(cs.T(), err, "should fail with missing payload") - header.View-- } -// TestSubmittingMultipleVotes tests that we can send multiple votes and they +// TestSubmittingMultipleVotes tests that we can send multiple blocks, and they // are queued and processed in expected way -func (cs *ComplianceSuite) TestSubmittingMultipleEntries() { - cs.hotstuff.On("Done", mock.Anything).Return(doneChan()) - +func (cs *EngineSuite) TestSubmittingMultipleEntries() { // create a vote - originID := unittest.IdentifierFixture() - voteCount := 15 + blockCount := 15 var wg sync.WaitGroup wg.Add(1) go func() { - for i := 0; i < voteCount; i++ { - vote := messages.BlockVote{ - BlockID: unittest.IdentifierFixture(), - View: rand.Uint64(), - SigData: unittest.SignatureFixture(), - } - cs.voteAggregator.On("AddVote", &model.Vote{ - View: vote.View, - BlockID: vote.BlockID, - SignerID: originID, - SigData: vote.SigData, - }).Return().Once() - // execute the vote submission - _ = cs.engine.Process(channels.ConsensusCommittee, originID, &vote) + for i := 0; i < blockCount; i++ { + block := unittest.BlockWithParentFixture(cs.head) + proposal := messages.NewBlockProposal(block) + cs.headerDB[block.Header.ParentID] = cs.head + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil).Once() + // execute the block submission + cs.engine.OnBlockProposal(flow.Slashable[messages.BlockProposal]{ + OriginID: unittest.IdentifierFixture(), + Message: proposal, + }) } wg.Done() }() wg.Add(1) go func() { // create a proposal that directly descends from the latest finalized header - originID := cs.participants[1].NodeID block := unittest.BlockWithParentFixture(cs.head) proposal := unittest.ProposalFromBlock(block) // store the data for retrieval cs.headerDB[block.Header.ParentID] = cs.head - cs.hotstuff.On("SubmitProposal", block.Header, cs.head.View).Return(doneChan()) - _ = cs.engine.Process(channels.ConsensusCommittee, originID, proposal) + hotstuffProposal := model.ProposalFromFlow(block.Header) + cs.hotstuff.On("SubmitProposal", hotstuffProposal).Return().Once() + cs.voteAggregator.On("AddBlock", hotstuffProposal).Once() + cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil).Once() + cs.engine.OnBlockProposal(flow.Slashable[messages.BlockProposal]{ + OriginID: unittest.IdentifierFixture(), + Message: proposal, + }) wg.Done() }() + // wait for all messages to be delivered to the engine message queue wg.Wait() - - time.Sleep(time.Second) - - // check that submit vote was called with correct parameters - cs.hotstuff.AssertExpectations(cs.T()) - cs.voteAggregator.AssertExpectations(cs.T()) -} - -// TestProcessUnsupportedMessageType tests that Process and ProcessLocal correctly handle a case where invalid message type -// was submitted from network layer. -func (cs *ComplianceSuite) TestProcessUnsupportedMessageType() { - invalidEvent := uint64(42) - err := cs.engine.Process("ch", unittest.IdentifierFixture(), invalidEvent) - // shouldn't result in error since byzantine inputs are expected - require.NoError(cs.T(), err) - // in case of local processing error cannot be consumed since all inputs are trusted - err = cs.engine.ProcessLocal(invalidEvent) - require.Error(cs.T(), err) - require.True(cs.T(), engine.IsIncompatibleInputTypeError(err)) + // wait for the votes queue to drain + assert.Eventually(cs.T(), func() bool { + return cs.engine.pendingBlocks.Len() == 0 + }, time.Second, time.Millisecond*10) } // TestOnFinalizedBlock tests if finalized block gets processed when send through `Engine`. // Tests the whole processing pipeline. -func (cs *ComplianceSuite) TestOnFinalizedBlock() { +func (cs *EngineSuite) TestOnFinalizedBlock() { finalizedBlock := unittest.BlockHeaderFixture() cs.head = finalizedBlock - - *cs.pending = modulemock.PendingBlockBuffer{} - cs.pending.On("PruneByView", finalizedBlock.View).Return(nil).Once() - cs.pending.On("Size").Return(uint(0)).Once() - cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock, finalizedBlock.View-1)) - - require.Eventually(cs.T(), - func() bool { - return cs.pending.AssertCalled(cs.T(), "PruneByView", finalizedBlock.View) - }, time.Second, time.Millisecond*20) + cs.headerDB[finalizedBlock.ID()] = finalizedBlock + + *cs.pending = *modulemock.NewPendingBlockBuffer(cs.T()) + // wait for both expected calls before ending the test + wg := new(sync.WaitGroup) + wg.Add(2) + cs.pending.On("PruneByView", finalizedBlock.View). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil).Once() + cs.pending.On("Size"). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(uint(0)).Once() + + cs.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) + unittest.AssertReturnsBefore(cs.T(), wg.Wait, time.Second, "an expected call to block buffer wasn't made") } diff --git a/engine/consensus/dkg/reactor_engine.go b/engine/consensus/dkg/reactor_engine.go index 0f2916b7ce2..1704483ef48 100644 --- a/engine/consensus/dkg/reactor_engine.go +++ b/engine/consensus/dkg/reactor_engine.go @@ -35,6 +35,7 @@ type dkgInfo struct { // ReactorEngine is an engine that reacts to chain events to start new DKG runs, // and manage subsequent phase transitions. Any unexpected error triggers a // panic as it would undermine the security of the protocol. +// TODO replace engine.Unit with component.Component type ReactorEngine struct { events.Noop unit *engine.Unit @@ -75,8 +76,7 @@ func NewReactorEngine( } // Ready implements the module ReadyDoneAware interface. It returns a channel -// that will close when the engine has successfully -// started. +// that will close when the engine has successfully started. func (e *ReactorEngine) Ready() <-chan struct{} { return e.unit.Ready(func() { // If we are starting up in the EpochSetup phase, try to start the DKG. @@ -87,28 +87,32 @@ func (e *ReactorEngine) Ready() <-chan struct{} { phase, err := snap.Phase() if err != nil { // unexpected storage-level error + // TODO use irrecoverable context e.log.Fatal().Err(err).Msg("failed to check epoch phase when starting DKG reactor engine") return } - if phase != flow.EpochPhaseSetup { - // start up in a non-setup phase - this is the typical path - return - } - currentCounter, err := snap.Epochs().Current().Counter() if err != nil { // unexpected storage-level error + // TODO use irrecoverable context e.log.Fatal().Err(err).Msg("failed to retrieve current epoch counter when starting DKG reactor engine") return } first, err := snap.Head() if err != nil { // unexpected storage-level error + // TODO use irrecoverable context e.log.Fatal().Err(err).Msg("failed to retrieve finalized header when starting DKG reactor engine") return } - e.startDKGForEpoch(currentCounter, first) + // If we start up in EpochSetup phase, attempt to start the DKG in case it wasn't started previously + if phase == flow.EpochPhaseSetup { + e.startDKGForEpoch(currentCounter, first) + } else if phase == flow.EpochPhaseCommitted { + // If we start up in EpochCommitted phase, ensure the DKG end state is set correctly. + e.handleEpochCommittedPhaseStarted(currentCounter, first) + } }) } @@ -120,19 +124,22 @@ func (e *ReactorEngine) Done() <-chan struct{} { // EpochSetupPhaseStarted handles the EpochSetupPhaseStarted protocol event by // starting the DKG process. +// NOTE: ReactorEngine will not recover from mid-DKG crashes, therefore we do not need to handle dropped protocol events here. func (e *ReactorEngine) EpochSetupPhaseStarted(currentEpochCounter uint64, first *flow.Header) { e.startDKGForEpoch(currentEpochCounter, first) } // EpochCommittedPhaseStarted handles the EpochCommittedPhaseStarted protocol // event by checking the consistency of our locally computed key share. +// NOTE: ReactorEngine will not recover from mid-DKG crashes, therefore we do not need to handle dropped protocol events here. func (e *ReactorEngine) EpochCommittedPhaseStarted(currentEpochCounter uint64, first *flow.Header) { e.handleEpochCommittedPhaseStarted(currentEpochCounter, first) } -// startDKGForEpoch starts the DKG instance for the given epoch, only if we have -// never started the DKG during setup phase for the given epoch. This allows consensus nodes which -// boot from a state snapshot within the EpochSetup phase to run the DKG. +// startDKGForEpoch attempts to start the DKG instance for the given epoch, +// only if we have never started the DKG during setup phase for the given epoch. +// This allows consensus nodes which boot from a state snapshot within the +// EpochSetup phase to run the DKG. // // It starts a new controller for the epoch and registers the triggers to regularly // query the DKG smart-contract and transition between phases at the specified views. @@ -151,6 +158,7 @@ func (e *ReactorEngine) startDKGForEpoch(currentEpochCounter uint64, first *flow started, err := e.dkgState.GetDKGStarted(nextEpochCounter) if err != nil { // unexpected storage-level error + // TODO use irrecoverable context log.Fatal().Err(err).Msg("could not check whether DKG is started") } if started { @@ -162,12 +170,14 @@ func (e *ReactorEngine) startDKGForEpoch(currentEpochCounter uint64, first *flow err = e.dkgState.SetDKGStarted(nextEpochCounter) if err != nil { // unexpected storage-level error + // TODO use irrecoverable context log.Fatal().Err(err).Msg("could not set dkg started") } curDKGInfo, err := e.getDKGInfo(firstID) if err != nil { // unexpected storage-level error + // TODO use irrecoverable context log.Fatal().Err(err).Msg("could not retrieve epoch info") } @@ -192,6 +202,7 @@ func (e *ReactorEngine) startDKGForEpoch(currentEpochCounter uint64, first *flow ) if err != nil { // no expected errors in controller factory + // TODO use irrecoverable context log.Fatal().Err(err).Msg("could not create DKG controller") } e.controller = controller @@ -273,6 +284,7 @@ func (e *ReactorEngine) handleEpochCommittedPhaseStarted(currentEpochCounter uin nextDKG, err := e.State.AtBlockID(firstBlock.ID()).Epochs().Next().DKG() if err != nil { // CAUTION: this should never happen, indicates a storage failure or corruption + // TODO use irrecoverable context log.Fatal().Err(err).Msg("checking beacon key consistency: could not retrieve next DKG info") return } @@ -282,16 +294,19 @@ func (e *ReactorEngine) handleEpochCommittedPhaseStarted(currentEpochCounter uin log.Warn().Msg("checking beacon key consistency: no key found") err := e.dkgState.SetDKGEndState(nextEpochCounter, flow.DKGEndStateNoKey) if err != nil { + // TODO use irrecoverable context log.Fatal().Err(err).Msg("failed to set dkg end state") } return } else if err != nil { + // TODO use irrecoverable context log.Fatal().Err(err).Msg("checking beacon key consistency: could not retrieve beacon private key for next epoch") return } nextDKGPubKey, err := nextDKG.KeyShare(e.me.NodeID()) if err != nil { + // TODO use irrecoverable context log.Fatal().Err(err).Msg("checking beacon key consistency: could not retrieve my beacon public key for next epoch") return } @@ -306,6 +321,7 @@ func (e *ReactorEngine) handleEpochCommittedPhaseStarted(currentEpochCounter uin Msg("checking beacon key consistency: locally computed beacon public key does not match beacon public key for next epoch") err := e.dkgState.SetDKGEndState(nextEpochCounter, flow.DKGEndStateInconsistentKey) if err != nil { + // TODO use irrecoverable context log.Fatal().Err(err).Msg("failed to set dkg end state") } return @@ -313,11 +329,13 @@ func (e *ReactorEngine) handleEpochCommittedPhaseStarted(currentEpochCounter uin err = e.dkgState.SetDKGEndState(nextEpochCounter, flow.DKGEndStateSuccess) if err != nil { - e.log.Fatal().Err(err).Msg("failed to set dkg") + // TODO use irrecoverable context + e.log.Fatal().Err(err).Msg("failed to set dkg end state") } log.Info().Msgf("successfully ended DKG, my beacon pub key for epoch %d is %s", nextEpochCounter, localPubKey) } +// TODO document error returns func (e *ReactorEngine) getDKGInfo(firstBlockID flow.Identifier) (*dkgInfo, error) { currEpoch := e.State.AtBlockID(firstBlockID).Epochs().Current() nextEpoch := e.State.AtBlockID(firstBlockID).Epochs().Next() @@ -387,6 +405,7 @@ func (e *ReactorEngine) registerPhaseTransition(view uint64, fromState dkgmodule log.Info().Msgf("ending %s...", fromState) err := phaseTransition() if err != nil { + // TODO use irrecoverable context log.Fatal().Err(err).Msgf("node failed to end %s", fromState) } log.Info().Msgf("ended %s successfully", fromState) diff --git a/engine/consensus/dkg/reactor_engine_test.go b/engine/consensus/dkg/reactor_engine_test.go index ba69b18615f..48e2707188d 100644 --- a/engine/consensus/dkg/reactor_engine_test.go +++ b/engine/consensus/dkg/reactor_engine_test.go @@ -273,6 +273,7 @@ type ReactorEngineSuite_CommittedPhase struct { dkgState *storage.DKGState state *protocol.State snap *protocol.Snapshot + factory *module.DKGControllerFactory engine *dkg.ReactorEngine } @@ -347,15 +348,17 @@ func (suite *ReactorEngineSuite_CommittedPhase) SetupTest() { suite.snap = new(protocol.Snapshot) suite.snap.On("Epochs").Return(epochQuery) + suite.snap.On("Head").Return(firstBlock, nil) suite.state = new(protocol.State) suite.state.On("AtBlockID", firstBlock.ID()).Return(suite.snap) + suite.state.On("Final").Return(suite.snap) // count number of warn-level logs suite.warnsLogged = 0 logger := hookedLogger(&suite.warnsLogged) - factory := new(module.DKGControllerFactory) + suite.factory = new(module.DKGControllerFactory) viewEvents := gadgets.NewViews() suite.engine = dkg.NewReactorEngine( @@ -363,7 +366,7 @@ func (suite *ReactorEngineSuite_CommittedPhase) SetupTest() { suite.me, suite.state, suite.dkgState, - factory, + suite.factory, viewEvents, ) } @@ -426,6 +429,102 @@ func (suite *ReactorEngineSuite_CommittedPhase) TestLocalDKGFailure() { suite.Assert().Equal(flow.DKGEndStateDKGFailure, suite.dkgEndState) } +// TestStartupInCommittedPhase_DKGSuccess tests that the dkg end state is correctly +// set when starting in EpochCommitted phase and a successful DKG +func (suite *ReactorEngineSuite_CommittedPhase) TestStartupInCommittedPhase_DKGSuccess() { + + // we are in the EpochSetup phase + suite.snap.On("Phase").Return(flow.EpochPhaseCommitted, nil).Once() + // the dkg for this epoch has been started but not ended + suite.dkgState.On("GetDKGStarted", suite.NextEpochCounter()).Return(true, nil).Once() + suite.dkgState.On("GetDKGEndState", suite.NextEpochCounter()).Return(flow.DKGEndStateUnknown, storerr.ErrNotFound).Once() + + // start up the engine + unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + + // we should not have instantiated the DKG + suite.factory.AssertNotCalled(suite.T(), "Create", + dkgmodule.CanonicalInstanceID(suite.firstBlock.ChainID, suite.NextEpochCounter()), + mock.Anything, + mock.Anything, + ) + // should set DKG end state + suite.Assert().Equal(flow.DKGEndStateSuccess, suite.dkgEndState) +} + +// TestStartupInCommittedPhase_DKGSuccess tests that the dkg end state is correctly +// set when starting in EpochCommitted phase and the DKG end state is already set. +func (suite *ReactorEngineSuite_CommittedPhase) TestStartupInCommittedPhase_DKGEndStateAlreadySet() { + + // we are in the EpochSetup phase + suite.snap.On("Phase").Return(flow.EpochPhaseCommitted, nil).Once() + // the dkg for this epoch has been started and ended + suite.dkgState.On("GetDKGStarted", suite.NextEpochCounter()).Return(true, nil).Once() + suite.dkgState.On("GetDKGEndState", suite.NextEpochCounter()).Return(flow.DKGEndStateNoKey, nil).Once() + + // start up the engine + unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + + // we should not have instantiated the DKG + suite.factory.AssertNotCalled(suite.T(), "Create", + dkgmodule.CanonicalInstanceID(suite.firstBlock.ChainID, suite.NextEpochCounter()), + mock.Anything, + mock.Anything, + ) +} + +// TestStartupInCommittedPhase_InconsistentKey tests that the dkg end state is correctly +// set when starting in EpochCommitted phase and we have stored an inconsistent key. +func (suite *ReactorEngineSuite_CommittedPhase) TestStartupInCommittedPhase_InconsistentKey() { + + // we are in the EpochSetup phase + suite.snap.On("Phase").Return(flow.EpochPhaseCommitted, nil).Once() + // the dkg for this epoch has been started but not ended + suite.dkgState.On("GetDKGStarted", suite.NextEpochCounter()).Return(true, nil).Once() + suite.dkgState.On("GetDKGEndState", suite.NextEpochCounter()).Return(flow.DKGEndStateUnknown, storerr.ErrNotFound).Once() + + // set our global pub key to a random value + suite.myGlobalBeaconPubKey = unittest.RandomBeaconPriv().PublicKey() + + // start up the engine + unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + + // we should not have instantiated the DKG + suite.factory.AssertNotCalled(suite.T(), "Create", + dkgmodule.CanonicalInstanceID(suite.firstBlock.ChainID, suite.NextEpochCounter()), + mock.Anything, + mock.Anything, + ) + // should set DKG end state + suite.Assert().Equal(flow.DKGEndStateInconsistentKey, suite.dkgEndState) +} + +// TestStartupInCommittedPhase_MissingKey tests that the dkg end state is correctly +// set when starting in EpochCommitted phase and we have not stored any key. +func (suite *ReactorEngineSuite_CommittedPhase) TestStartupInCommittedPhase_MissingKey() { + + // we are in the EpochSetup phase + suite.snap.On("Phase").Return(flow.EpochPhaseCommitted, nil).Once() + // the dkg for this epoch has been started but not ended + suite.dkgState.On("GetDKGStarted", suite.NextEpochCounter()).Return(true, nil).Once() + suite.dkgState.On("GetDKGEndState", suite.NextEpochCounter()).Return(flow.DKGEndStateUnknown, storerr.ErrNotFound).Once() + + // remove our key + suite.myLocalBeaconKey = nil + + // start up the engine + unittest.AssertClosesBefore(suite.T(), suite.engine.Ready(), time.Second) + + // we should not have instantiated the DKG + suite.factory.AssertNotCalled(suite.T(), "Create", + dkgmodule.CanonicalInstanceID(suite.firstBlock.ChainID, suite.NextEpochCounter()), + mock.Anything, + mock.Anything, + ) + // should set DKG end state + suite.Assert().Equal(flow.DKGEndStateNoKey, suite.dkgEndState) +} + // utility function to track the number of warn-level calls to a logger func hookedLogger(calls *int) zerolog.Logger { hook := zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) { diff --git a/engine/consensus/ingestion/core.go b/engine/consensus/ingestion/core.go index 64408355eb0..de4f4a1e122 100644 --- a/engine/consensus/ingestion/core.go +++ b/engine/consensus/ingestion/core.go @@ -10,7 +10,7 @@ import ( "github.com/rs/zerolog" "go.opentelemetry.io/otel/attribute" - "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -185,7 +185,7 @@ func (e *Core) validateGuarantors(guarantee *flow.CollectionGuarantee) error { } // determine whether signers reach minimally required stake threshold - threshold := hotstuff.ComputeWeightThresholdForBuildingQC(clusterMembers.TotalWeight()) // compute required stake threshold + threshold := committees.WeightThresholdToBuildQC(clusterMembers.TotalWeight()) // compute required stake threshold totalStake := flow.IdentityList(guarantors).TotalWeight() if totalStake < threshold { return engine.NewInvalidInputErrorf("collection guarantee qc signers have insufficient stake of %d (required=%d)", totalStake, threshold) diff --git a/engine/consensus/matching/engine_test.go b/engine/consensus/matching/engine_test.go index 312aca7af33..170e633da86 100644 --- a/engine/consensus/matching/engine_test.go +++ b/engine/consensus/matching/engine_test.go @@ -67,7 +67,7 @@ func (s *MatchingEngineSuite) TestOnFinalizedBlock() { finalizedBlock := unittest.BlockHeaderFixture() s.state.On("Final").Return(unittest.StateSnapshotForKnownBlock(finalizedBlock, nil)) s.core.On("OnBlockFinalization").Return(nil).Once() - s.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock, finalizedBlock.View-1)) + s.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) // matching engine has at least 100ms ticks for processing events time.Sleep(1 * time.Second) @@ -93,7 +93,7 @@ func (s *MatchingEngineSuite) TestOnBlockIncorporated() { } s.index.On("ByBlockID", incorporatedBlockID).Return(index, nil) - s.engine.OnBlockIncorporated(model.BlockFromFlow(incorporatedBlock, incorporatedBlock.View-1)) + s.engine.OnBlockIncorporated(model.BlockFromFlow(incorporatedBlock)) // matching engine has at least 100ms ticks for processing events time.Sleep(1 * time.Second) diff --git a/engine/consensus/message_hub/message_hub.go b/engine/consensus/message_hub/message_hub.go new file mode 100644 index 00000000000..1ca8dca8787 --- /dev/null +++ b/engine/consensus/message_hub/message_hub.go @@ -0,0 +1,510 @@ +package message_hub + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/consensus/hotstuff/notifications" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/common/fifoqueue" + "github.com/onflow/flow-go/engine/consensus" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/logging" +) + +// defaultMessageHubRequestsWorkers number of workers to dispatch events for requests +const defaultMessageHubRequestsWorkers = 5 + +// defaultProposalQueueCapacity number of pending outgoing proposals stored in queue +const defaultProposalQueueCapacity = 3 + +// defaultVoteQueueCapacity number of pending outgoing votes stored in queue +const defaultVoteQueueCapacity = 20 + +// defaultTimeoutQueueCapacity number of pending outgoing timeouts stored in queue +const defaultTimeoutQueueCapacity = 3 + +// packedVote is a helper structure to pack recipientID and vote into one structure to pass through fifoqueue.FifoQueue +type packedVote struct { + recipientID flow.Identifier + vote *messages.BlockVote +} + +// MessageHub is a central module for handling incoming and outgoing messages via consensus channel. +// It performs message routing for incoming messages by matching them by type and sending to respective engine. +// For incoming messages handling processing looks like this: +// +// +-------------------+ +------------+ +// -->| Consensus-Channel |----->| MessageHub | +// +-------------------+ +------+-----+ +// ------------|------------ +// +------+---------+ | +------+-----+ | +------+------------+ +// | VoteAggregator |----+ | Compliance | +----| TimeoutAggregator | +// +----------------+ +------------+ +------+------------+ +// vote block timeout object +// +// MessageHub acts as communicator and handles hotstuff.Consumer communication events to send votes, broadcast timeouts +// and proposals. It is responsible for communication between consensus participants. +// It implements hotstuff.Consumer interface and needs to be subscribed for notifications via pub/sub. +// All communicator events are handled on worker thread to prevent sender from blocking. +// For outgoing messages processing logic looks like this: +// +// +-------------------+ +------------+ +----------+ +------------------------+ +// | Consensus-Channel |<-----| MessageHub |<-----| Consumer |<-----| Hotstuff | +// +-------------------+ +------+-----+ +----------+ +------------------------+ +// pub/sub vote, timeout, proposal +// +// MessageHub is safe to use in concurrent environment. +type MessageHub struct { + *component.ComponentManager + notifications.NoopConsumer + log zerolog.Logger + me module.Local + engineMetrics module.EngineMetrics + state protocol.State + payloads storage.Payloads + con network.Conduit + pushBlocksCon network.Conduit + ownOutboundMessageNotifier engine.Notifier + ownOutboundVotes *fifoqueue.FifoQueue // queue for handling outgoing vote transmissions + ownOutboundProposals *fifoqueue.FifoQueue // queue for handling outgoing proposal transmissions + ownOutboundTimeouts *fifoqueue.FifoQueue // queue for handling outgoing timeout transmissions + + // injected dependencies + compliance consensus.Compliance // handler of incoming block proposals + hotstuff module.HotStuff // used to submit proposals that were previously broadcast + voteAggregator hotstuff.VoteAggregator // handler of incoming votes + timeoutAggregator hotstuff.TimeoutAggregator // handler of incoming timeouts +} + +var _ network.MessageProcessor = (*MessageHub)(nil) +var _ hotstuff.CommunicatorConsumer = (*MessageHub)(nil) + +// NewMessageHub constructs new instance of message hub +// No errors are expected during normal operations. +func NewMessageHub(log zerolog.Logger, + engineMetrics module.EngineMetrics, + net network.Network, + me module.Local, + compliance consensus.Compliance, + hotstuff module.HotStuff, + voteAggregator hotstuff.VoteAggregator, + timeoutAggregator hotstuff.TimeoutAggregator, + state protocol.State, + payloads storage.Payloads, +) (*MessageHub, error) { + ownOutboundVotes, err := fifoqueue.NewFifoQueue(defaultVoteQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize votes queue") + } + ownOutboundProposals, err := fifoqueue.NewFifoQueue(defaultProposalQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize blocks queue") + } + ownOutboundTimeouts, err := fifoqueue.NewFifoQueue(defaultTimeoutQueueCapacity) + if err != nil { + return nil, fmt.Errorf("could not initialize timeouts queue") + } + hub := &MessageHub{ + log: log.With().Str("engine", "message_hub").Logger(), + me: me, + engineMetrics: engineMetrics, + state: state, + payloads: payloads, + compliance: compliance, + hotstuff: hotstuff, + voteAggregator: voteAggregator, + timeoutAggregator: timeoutAggregator, + ownOutboundMessageNotifier: engine.NewNotifier(), + ownOutboundVotes: ownOutboundVotes, + ownOutboundProposals: ownOutboundProposals, + ownOutboundTimeouts: ownOutboundTimeouts, + } + + // register with the network layer and store the conduit + hub.con, err = net.Register(channels.ConsensusCommittee, hub) + if err != nil { + return nil, fmt.Errorf("could not register core: %w", err) + } + + // register with the network layer and store the conduit + hub.pushBlocksCon, err = net.Register(channels.PushBlocks, hub) + if err != nil { + return nil, fmt.Errorf("could not register engine: %w", err) + } + + componentBuilder := component.NewComponentManagerBuilder() + // This implementation tolerates if the networking layer sometimes blocks on send requests. + // We use by default 5 go-routines here. This is fine, because outbound messages are temporally sparse + // under normal operations. Hence, the go-routines should mostly be asleep waiting for work. + for i := 0; i < defaultMessageHubRequestsWorkers; i++ { + componentBuilder.AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + hub.queuedMessagesProcessingLoop(ctx) + }) + } + hub.ComponentManager = componentBuilder.Build() + return hub, nil +} + +// queuedMessagesProcessingLoop orchestrates dispatching of previously queued messages +func (h *MessageHub) queuedMessagesProcessingLoop(ctx irrecoverable.SignalerContext) { + notifier := h.ownOutboundMessageNotifier.Channel() + for { + select { + case <-ctx.Done(): + return + case <-notifier: + err := h.sendOwnMessages(ctx) + if err != nil { + ctx.Throw(fmt.Errorf("internal error processing queued messages: %w", err)) + return + } + } + } +} + +// sendOwnMessages is a function which dispatches previously queued messages on worker thread +// This function is called whenever we have queued messages ready to be dispatched. +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnMessages(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + default: + } + + msg, ok := h.ownOutboundProposals.Pop() + if ok { + block := msg.(*flow.Header) + err := h.sendOwnProposal(block) + if err != nil { + return fmt.Errorf("could not process queued block %v: %w", block.ID(), err) + } + continue + } + + msg, ok = h.ownOutboundVotes.Pop() + if ok { + packed := msg.(*packedVote) + err := h.sendOwnVote(packed) + if err != nil { + return fmt.Errorf("could not process queued vote: %w", err) + } + continue + } + + msg, ok = h.ownOutboundTimeouts.Pop() + if ok { + err := h.sendOwnTimeout(msg.(*model.TimeoutObject)) + if err != nil { + return fmt.Errorf("coult not process queued timeout: %w", err) + } + continue + } + + // when there is no more messages in the queue, back to the loop to wait + // for the next incoming message to arrive. + return nil + } +} + +// sendOwnTimeout propagates the timeout to the consensus committee (excluding myself) +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnTimeout(timeout *model.TimeoutObject) error { + log := timeout.LogContext(h.log).Logger() + log.Info().Msg("processing timeout broadcast request from hotstuff") + + // Retrieve all consensus nodes (excluding myself). + // CAUTION: We must include also nodes with weight zero, because otherwise + // TCs might not be constructed at epoch switchover. + recipients, err := h.state.Final().Identities(filter.And( + filter.Not(filter.Ejected), + filter.HasRole(flow.RoleConsensus), + filter.Not(filter.HasNodeID(h.me.NodeID())), + )) + if err != nil { + return fmt.Errorf("could not get consensus recipients for broadcasting timeout: %w", err) + } + + // create the timeout message + msg := &messages.TimeoutObject{ + View: timeout.View, + NewestQC: timeout.NewestQC, + LastViewTC: timeout.LastViewTC, + SigData: timeout.SigData, + TimeoutTick: timeout.TimeoutTick, + } + err = h.con.Publish(msg, recipients.NodeIDs()...) + if err != nil { + if !errors.Is(err, network.EmptyTargetList) { + log.Err(err).Msg("could not broadcast timeout") + } + return nil + } + log.Info().Msg("consensus timeout was broadcast") + h.engineMetrics.MessageSent(metrics.EngineConsensusMessageHub, metrics.MessageTimeoutObject) + + return nil +} + +// sendOwnVote propagates the vote via unicast to another node that is the next leader +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnVote(packed *packedVote) error { + log := h.log.With(). + Hex("block_id", packed.vote.BlockID[:]). + Uint64("block_view", packed.vote.View). + Hex("recipient_id", packed.recipientID[:]). + Logger() + log.Info().Msg("processing vote transmission request from hotstuff") + + // send the vote the desired recipient + err := h.con.Unicast(packed.vote, packed.recipientID) + if err != nil { + log.Err(err).Msg("could not send vote") + return nil + } + h.engineMetrics.MessageSent(metrics.EngineConsensusMessageHub, metrics.MessageBlockVote) + log.Info().Msg("block vote transmitted") + + return nil +} + +// sendOwnProposal propagates the block proposal to the consensus committee and submits to non-consensus network: +// - broadcast to all other consensus participants (excluding myself) +// - broadcast to all non-consensus participants +// +// No errors are expected during normal operations. +func (h *MessageHub) sendOwnProposal(header *flow.Header) error { + // first, check that we are the proposer of the block + if header.ProposerID != h.me.NodeID() { + return fmt.Errorf("cannot broadcast proposal with non-local proposer (%x)", header.ProposerID) + } + + // retrieve the payload for the block + payload, err := h.payloads.ByBlockID(header.ID()) + if err != nil { + return fmt.Errorf("could not retrieve payload for proposal: %w", err) + } + + log := h.log.With(). + Str("chain_id", header.ChainID.String()). + Uint64("block_height", header.Height). + Uint64("block_view", header.View). + Hex("block_id", logging.Entity(header)). + Hex("parent_id", header.ParentID[:]). + Hex("payload_hash", header.PayloadHash[:]). + Int("guarantees_count", len(payload.Guarantees)). + Int("seals_count", len(payload.Seals)). + Int("receipts_count", len(payload.Receipts)). + Time("timestamp", header.Timestamp). + Hex("signers", header.ParentVoterIndices). + //Dur("delay", delay). + Logger() + + log.Debug().Msg("processing proposal broadcast request from hotstuff") + + // Retrieve all consensus nodes (excluding myself). + // CAUTION: We must include also nodes with weight zero, because otherwise + // new consensus nodes for the next epoch are left out. + // Note: retrieving the final state requires a time-intensive database read. + // Therefore, we execute this in a separate routine, because + // `OnOwnTimeout` is directly called by the consensus core logic. + allIdentities, err := h.state.AtBlockID(header.ParentID).Identities(filter.And( + filter.Not(filter.Ejected), + filter.Not(filter.HasNodeID(h.me.NodeID())), + )) + if err != nil { + return fmt.Errorf("could not get identities for broadcasting proposal: %w", err) + } + + consRecipients := allIdentities.Filter(filter.HasRole(flow.RoleConsensus)) + + // NOTE: some fields are not needed for the message + // - proposer ID is conveyed over the network message + // - the payload hash is deduced from the payload + proposal := messages.NewBlockProposal(&flow.Block{ + Header: header, + Payload: payload, + }) + + // broadcast the proposal to consensus nodes + err = h.con.Publish(proposal, consRecipients.NodeIDs()...) + if err != nil { + if !errors.Is(err, network.EmptyTargetList) { + log.Err(err).Msg("could not send proposal message") + } + return nil + } + log.Info().Msg("block proposal was broadcast") + + // submit proposal to non-consensus nodes + h.provideProposal(proposal, allIdentities.Filter(filter.Not(filter.HasRole(flow.RoleConsensus)))) + h.engineMetrics.MessageSent(metrics.EngineConsensusMessageHub, metrics.MessageBlockProposal) + + return nil +} + +// provideProposal is used when we want to broadcast a local block to the rest of the +// network (non-consensus nodes). +func (h *MessageHub) provideProposal(proposal *messages.BlockProposal, recipients flow.IdentityList) { + header := proposal.Block.Header + blockID := header.ID() + log := h.log.With(). + Uint64("block_view", header.View). + Hex("block_id", blockID[:]). + Hex("parent_id", header.ParentID[:]). + Logger() + log.Info().Msg("block proposal submitted for propagation") + + // submit the block to the targets + err := h.pushBlocksCon.Publish(proposal, recipients.NodeIDs()...) + if err != nil { + h.log.Err(err).Msg("failed to broadcast block") + return + } + + log.Info().Msg("block proposal propagated to non-consensus nodes") +} + +// OnOwnVote propagates the vote to relevant recipient(s): +// - [common case] vote is queued and is sent via unicast to another node that is the next leader by worker +// - [special case] this node is the next leader: vote is directly forwarded to the node's internal `VoteAggregator` +func (h *MessageHub) OnOwnVote(blockID flow.Identifier, view uint64, sigData []byte, recipientID flow.Identifier) { + vote := &messages.BlockVote{ + BlockID: blockID, + View: view, + SigData: sigData, + } + + // special case: I am the next leader + if recipientID == h.me.NodeID() { + h.forwardToOwnVoteAggregator(vote, h.me.NodeID()) // forward vote to my own `voteAggregator` + return + } + + // common case: someone else is leader + packed := &packedVote{ + recipientID: recipientID, + vote: vote, + } + if ok := h.ownOutboundVotes.Push(packed); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineConsensusMessageHub, metrics.MessageBlockVote) + } +} + +// OnOwnTimeout forwards timeout to node's internal `timeoutAggregator` and queues timeout for +// subsequent propagation to all consensus participants (excluding this node) +func (h *MessageHub) OnOwnTimeout(timeout *model.TimeoutObject) { + h.forwardToOwnTimeoutAggregator(timeout) // forward timeout to my own `timeoutAggregator` + if ok := h.ownOutboundTimeouts.Push(timeout); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineConsensusMessageHub, metrics.MessageTimeoutObject) + } +} + +// OnOwnProposal directly forwards proposal to HotStuff core logic (skipping compliance engine as we assume our +// own proposals to be correct) and queues proposal for subsequent propagation to all consensus participants (including this node). +// The proposal will only be placed in the queue, after the specified delay (or dropped on shutdown signal). +func (h *MessageHub) OnOwnProposal(proposal *flow.Header, targetPublicationTime time.Time) { + go func() { + select { + case <-time.After(time.Until(targetPublicationTime)): + case <-h.ShutdownSignal(): + return + } + + hotstuffProposal := model.ProposalFromFlow(proposal) + // notify vote aggregator that new block proposal is available, in case we are next leader + h.voteAggregator.AddBlock(hotstuffProposal) // non-blocking + + // TODO(active-pacemaker): replace with pub/sub? + // submit proposal to our own processing pipeline + h.hotstuff.SubmitProposal(hotstuffProposal) // non-blocking + + if ok := h.ownOutboundProposals.Push(proposal); ok { + h.ownOutboundMessageNotifier.Notify() + } else { + h.engineMetrics.OutboundMessageDropped(metrics.EngineConsensusMessageHub, metrics.MessageBlockProposal) + } + }() +} + +// Process handles incoming messages from consensus channel. After matching message by type, sends it to the correct +// component for handling. +// No errors are expected during normal operations. +func (h *MessageHub) Process(channel channels.Channel, originID flow.Identifier, message interface{}) error { + switch msg := message.(type) { + case *messages.BlockProposal: + h.compliance.OnBlockProposal(flow.Slashable[messages.BlockProposal]{ + OriginID: originID, + Message: msg, + }) + case *messages.BlockVote: + h.forwardToOwnVoteAggregator(msg, originID) + case *messages.TimeoutObject: + t := &model.TimeoutObject{ + View: msg.View, + NewestQC: msg.NewestQC, + LastViewTC: msg.LastViewTC, + SignerID: originID, + SigData: msg.SigData, + TimeoutTick: msg.TimeoutTick, + } + h.forwardToOwnTimeoutAggregator(t) + default: + h.log.Warn().Msgf("%v delivered unsupported message %T through %v", originID, message, channel) + } + return nil +} + +// forwardToOwnVoteAggregator converts vote to generic `model.Vote`, logs vote and forwards it to own `voteAggregator`. +// Per API convention, timeoutAggregator` is non-blocking, hence, this call returns quickly. +func (h *MessageHub) forwardToOwnVoteAggregator(vote *messages.BlockVote, originID flow.Identifier) { + h.engineMetrics.MessageReceived(metrics.EngineConsensusMessageHub, metrics.MessageBlockVote) + v := &model.Vote{ + View: vote.View, + BlockID: vote.BlockID, + SignerID: originID, + SigData: vote.SigData, + } + h.log.Info(). + Uint64("block_view", v.View). + Hex("block_id", v.BlockID[:]). + Hex("voter", v.SignerID[:]). + Str("vote_id", v.ID().String()). + Msg("block vote received, forwarding block vote to hotstuff vote aggregator") + h.voteAggregator.AddVote(v) +} + +// forwardToOwnTimeoutAggregator logs timeout and forwards it to own `timeoutAggregator`. +// Per API convention, timeoutAggregator` is non-blocking, hence, this call returns quickly. +func (h *MessageHub) forwardToOwnTimeoutAggregator(t *model.TimeoutObject) { + h.engineMetrics.MessageReceived(metrics.EngineConsensusMessageHub, metrics.MessageTimeoutObject) + h.log.Info(). + Hex("origin_id", t.SignerID[:]). + Uint64("view", t.View). + Str("timeout_id", t.ID().String()). + Msg("timeout received, forwarding timeout to hotstuff timeout aggregator") + h.timeoutAggregator.AddTimeout(t) +} diff --git a/engine/consensus/message_hub/message_hub_test.go b/engine/consensus/message_hub/message_hub_test.go new file mode 100644 index 00000000000..97351ba649b --- /dev/null +++ b/engine/consensus/message_hub/message_hub_test.go @@ -0,0 +1,339 @@ +package message_hub + +import ( + "context" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/onflow/flow-go/consensus/hotstuff/helper" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" + mockconsensus "github.com/onflow/flow-go/engine/consensus/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/module/util" + netint "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/mocknetwork" + protint "github.com/onflow/flow-go/state/protocol" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storerr "github.com/onflow/flow-go/storage" + storage "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +func TestMessageHub(t *testing.T) { + suite.Run(t, new(MessageHubSuite)) +} + +// MessageHubSuite tests the consensus message hub. Holds mocked dependencies that are used by different test scenarios. +type MessageHubSuite struct { + suite.Suite + + // parameters + participants flow.IdentityList + myID flow.Identifier + head *flow.Header + + // mocked dependencies + payloads *storage.Payloads + me *module.Local + state *protocol.MutableState + net *mocknetwork.Network + con *mocknetwork.Conduit + pushBlocksCon *mocknetwork.Conduit + hotstuff *module.HotStuff + voteAggregator *hotstuff.VoteAggregator + timeoutAggregator *hotstuff.TimeoutAggregator + compliance *mockconsensus.Compliance + snapshot *protocol.Snapshot + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + errs <-chan error + hub *MessageHub +} + +func (s *MessageHubSuite) SetupTest() { + // seed the RNG + rand.Seed(time.Now().UnixNano()) + + // initialize the paramaters + s.participants = unittest.IdentityListFixture(3, + unittest.WithRole(flow.RoleConsensus), + unittest.WithWeight(1000), + ) + s.myID = s.participants[0].NodeID + block := unittest.BlockFixture() + s.head = block.Header + + s.payloads = storage.NewPayloads(s.T()) + s.me = module.NewLocal(s.T()) + s.state = protocol.NewMutableState(s.T()) + s.net = mocknetwork.NewNetwork(s.T()) + s.con = mocknetwork.NewConduit(s.T()) + s.pushBlocksCon = mocknetwork.NewConduit(s.T()) + s.hotstuff = module.NewHotStuff(s.T()) + s.voteAggregator = hotstuff.NewVoteAggregator(s.T()) + s.timeoutAggregator = hotstuff.NewTimeoutAggregator(s.T()) + s.compliance = mockconsensus.NewCompliance(s.T()) + + // set up protocol state mock + s.state = &protocol.MutableState{} + s.state.On("Final").Return( + func() protint.Snapshot { + return s.snapshot + }, + ) + s.state.On("AtBlockID", mock.Anything).Return( + func(blockID flow.Identifier) protint.Snapshot { + return s.snapshot + }, + ) + + // set up local module mock + s.me.On("NodeID").Return( + func() flow.Identifier { + return s.myID + }, + ).Maybe() + + // set up network module mock + s.net.On("Register", mock.Anything, mock.Anything).Return( + func(channel channels.Channel, engine netint.MessageProcessor) netint.Conduit { + if channel == channels.ConsensusCommittee { + return s.con + } else if channel == channels.PushBlocks { + return s.pushBlocksCon + } else { + s.T().Fail() + } + return nil + }, + nil, + ) + + // set up protocol snapshot mock + s.snapshot = &protocol.Snapshot{} + s.snapshot.On("Identities", mock.Anything).Return( + func(filter flow.IdentityFilter) flow.IdentityList { + return s.participants.Filter(filter) + }, + nil, + ) + s.snapshot.On("Head").Return( + func() *flow.Header { + return s.head + }, + nil, + ) + + engineMetrics := metrics.NewNoopCollector() + hub, err := NewMessageHub( + unittest.Logger(), + engineMetrics, + s.net, + s.me, + s.compliance, + s.hotstuff, + s.voteAggregator, + s.timeoutAggregator, + s.state, + s.payloads, + ) + require.NoError(s.T(), err) + s.hub = hub + + s.ctx, s.cancel, s.errs = irrecoverable.WithSignallerAndCancel(context.Background()) + s.hub.Start(s.ctx) + + unittest.AssertClosesBefore(s.T(), s.hub.Ready(), time.Second) +} + +// TearDownTest stops the hub and checks there are no errors thrown to the SignallerContext. +func (s *MessageHubSuite) TearDownTest() { + s.cancel() + unittest.RequireCloseBefore(s.T(), s.hub.Done(), time.Second, "hub failed to stop") + select { + case err := <-s.errs: + assert.NoError(s.T(), err) + default: + } +} + +// TestProcessIncomingMessages tests processing of incoming messages, MessageHub matches messages by type +// and sends them to other modules which execute business logic. +func (s *MessageHubSuite) TestProcessIncomingMessages() { + var channel channels.Channel + originID := unittest.IdentifierFixture() + s.Run("to-compliance-engine", func() { + block := unittest.BlockFixture() + + blockProposalMsg := messages.NewBlockProposal(&block) + expectedComplianceMsg := flow.Slashable[messages.BlockProposal]{ + OriginID: originID, + Message: blockProposalMsg, + } + s.compliance.On("OnBlockProposal", expectedComplianceMsg).Return(nil).Once() + err := s.hub.Process(channel, originID, blockProposalMsg) + require.NoError(s.T(), err) + }) + s.Run("to-vote-aggregator", func() { + expectedVote := unittest.VoteFixture(unittest.WithVoteSignerID(originID)) + msg := &messages.BlockVote{ + View: expectedVote.View, + BlockID: expectedVote.BlockID, + SigData: expectedVote.SigData, + } + s.voteAggregator.On("AddVote", expectedVote) + err := s.hub.Process(channel, originID, msg) + require.NoError(s.T(), err) + }) + s.Run("to-timeout-aggregator", func() { + expectedTimeout := helper.TimeoutObjectFixture(helper.WithTimeoutObjectSignerID(originID)) + msg := &messages.TimeoutObject{ + View: expectedTimeout.View, + NewestQC: expectedTimeout.NewestQC, + LastViewTC: expectedTimeout.LastViewTC, + SigData: expectedTimeout.SigData, + } + s.timeoutAggregator.On("AddTimeout", expectedTimeout) + err := s.hub.Process(channel, originID, msg) + require.NoError(s.T(), err) + }) + s.Run("unsupported-msg-type", func() { + err := s.hub.Process(channel, originID, struct{}{}) + require.NoError(s.T(), err) + }) +} + +// TestOnOwnProposal tests broadcasting proposals with different inputs +func (s *MessageHubSuite) TestOnOwnProposal() { + // add execution node to participants to make sure we exclude them from broadcast + s.participants = append(s.participants, unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution))) + + // generate a parent with height and chain ID set + parent := unittest.BlockHeaderFixture() + parent.ChainID = "test" + parent.Height = 10 + + // create a block with the parent and store the payload with correct ID + block := unittest.BlockWithParentFixture(parent) + block.Header.ProposerID = s.myID + + s.payloads.On("ByBlockID", block.Header.ID()).Return(block.Payload, nil) + s.payloads.On("ByBlockID", mock.Anything).Return(nil, storerr.ErrNotFound) + + s.Run("should fail with wrong proposer", func() { + header := *block.Header + header.ProposerID = unittest.IdentifierFixture() + err := s.hub.sendOwnProposal(&header) + require.Error(s.T(), err, "should fail with wrong proposer") + header.ProposerID = s.myID + }) + + // should fail with wrong block ID (payload unavailable) + s.Run("should fail with wrong block ID", func() { + header := *block.Header + header.View++ + err := s.hub.sendOwnProposal(&header) + require.Error(s.T(), err, "should fail with missing payload") + header.View-- + }) + + s.Run("should broadcast proposal and pass to HotStuff for valid proposals", func() { + expectedBroadcastMsg := messages.NewBlockProposal(block) + + submitted := make(chan struct{}) // closed when proposal is submitted to hotstuff + hotstuffProposal := model.ProposalFromFlow(block.Header) + s.voteAggregator.On("AddBlock", hotstuffProposal).Once() + s.hotstuff.On("SubmitProposal", hotstuffProposal). + Run(func(args mock.Arguments) { close(submitted) }). + Once() + + broadcast := make(chan struct{}) // closed when proposal is broadcast + s.con.On("Publish", expectedBroadcastMsg, s.participants[1].NodeID, s.participants[2].NodeID). + Run(func(_ mock.Arguments) { close(broadcast) }). + Return(nil). + Once() + + s.pushBlocksCon.On("Publish", expectedBroadcastMsg, s.participants[3].NodeID).Return(nil) + + // submit to broadcast proposal + s.hub.OnOwnProposal(block.Header, time.Now()) + + unittest.AssertClosesBefore(s.T(), util.AllClosed(broadcast, submitted), time.Second) + }) +} + +// TestProcessMultipleMessagesHappyPath tests submitting all types of messages through full processing pipeline and +// asserting that expected message transmissions happened as expected. +func (s *MessageHubSuite) TestProcessMultipleMessagesHappyPath() { + var wg sync.WaitGroup + + // add execution node to participants to make sure we exclude them from broadcast + s.participants = append(s.participants, unittest.IdentityFixture(unittest.WithRole(flow.RoleExecution))) + + s.Run("vote", func() { + wg.Add(1) + // prepare vote fixture + vote := unittest.VoteFixture() + recipientID := unittest.IdentifierFixture() + s.con.On("Unicast", mock.Anything, recipientID).Run(func(_ mock.Arguments) { + wg.Done() + }).Return(nil) + + // submit vote + s.hub.OnOwnVote(vote.BlockID, vote.View, vote.SigData, recipientID) + }) + s.Run("timeout", func() { + wg.Add(1) + // prepare timeout fixture + timeout := helper.TimeoutObjectFixture() + expectedBroadcastMsg := &messages.TimeoutObject{ + View: timeout.View, + NewestQC: timeout.NewestQC, + LastViewTC: timeout.LastViewTC, + SigData: timeout.SigData, + } + s.con.On("Publish", expectedBroadcastMsg, s.participants[1].NodeID, s.participants[2].NodeID). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil) + s.timeoutAggregator.On("AddTimeout", timeout).Once() + // submit timeout + s.hub.OnOwnTimeout(timeout) + }) + s.Run("proposal", func() { + wg.Add(1) + // prepare proposal fixture + proposal := unittest.BlockWithParentAndProposerFixture(s.T(), s.head, s.myID) + s.payloads.On("ByBlockID", proposal.Header.ID()).Return(proposal.Payload, nil) + + // unset chain and height to make sure they are correctly reconstructed + hotstuffProposal := model.ProposalFromFlow(proposal.Header) + s.voteAggregator.On("AddBlock", hotstuffProposal).Once() + s.hotstuff.On("SubmitProposal", hotstuffProposal) + expectedBroadcastMsg := messages.NewBlockProposal(&proposal) + s.con.On("Publish", expectedBroadcastMsg, s.participants[1].NodeID, s.participants[2].NodeID). + Run(func(_ mock.Arguments) { wg.Done() }). + Return(nil) + s.pushBlocksCon.On("Publish", expectedBroadcastMsg, s.participants[3].NodeID).Return(nil) + + // submit proposal + s.hub.OnOwnProposal(proposal.Header, time.Now()) + }) + + unittest.RequireReturnsBefore(s.T(), func() { + wg.Wait() + }, time.Second, "expect to process messages before timeout") +} diff --git a/engine/consensus/mock/compliance.go b/engine/consensus/mock/compliance.go new file mode 100644 index 00000000000..a4715c05c8b --- /dev/null +++ b/engine/consensus/mock/compliance.go @@ -0,0 +1,79 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" + + messages "github.com/onflow/flow-go/model/messages" + + mock "github.com/stretchr/testify/mock" +) + +// Compliance is an autogenerated mock type for the Compliance type +type Compliance struct { + mock.Mock +} + +// Done provides a mock function with given fields: +func (_m *Compliance) Done() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// OnBlockProposal provides a mock function with given fields: proposal +func (_m *Compliance) OnBlockProposal(proposal flow.Slashable[messages.BlockProposal]) { + _m.Called(proposal) +} + +// OnSyncedBlock provides a mock function with given fields: block +func (_m *Compliance) OnSyncedBlock(block flow.Slashable[messages.BlockProposal]) { + _m.Called(block) +} + +// Ready provides a mock function with given fields: +func (_m *Compliance) Ready() <-chan struct{} { + ret := _m.Called() + + var r0 <-chan struct{} + if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(<-chan struct{}) + } + } + + return r0 +} + +// Start provides a mock function with given fields: _a0 +func (_m *Compliance) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} + +type mockConstructorTestingTNewCompliance interface { + mock.TestingT + Cleanup(func()) +} + +// NewCompliance creates a new instance of Compliance. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCompliance(t mockConstructorTestingTNewCompliance) *Compliance { + mock := &Compliance{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/consensus/mock/proposal_provider.go b/engine/consensus/mock/proposal_provider.go new file mode 100644 index 00000000000..b53cef236e1 --- /dev/null +++ b/engine/consensus/mock/proposal_provider.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mock + +import ( + messages "github.com/onflow/flow-go/model/messages" + mock "github.com/stretchr/testify/mock" +) + +// ProposalProvider is an autogenerated mock type for the ProposalProvider type +type ProposalProvider struct { + mock.Mock +} + +// ProvideProposal provides a mock function with given fields: proposal +func (_m *ProposalProvider) ProvideProposal(proposal *messages.BlockProposal) { + _m.Called(proposal) +} + +type mockConstructorTestingTNewProposalProvider interface { + mock.TestingT + Cleanup(func()) +} + +// NewProposalProvider creates a new instance of ProposalProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProposalProvider(t mockConstructorTestingTNewProposalProvider) *ProposalProvider { + mock := &ProposalProvider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/consensus/provider/engine.go b/engine/consensus/provider/engine.go deleted file mode 100644 index f4178c84070..00000000000 --- a/engine/consensus/provider/engine.go +++ /dev/null @@ -1,169 +0,0 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - -package provider - -import ( - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" - "github.com/onflow/flow-go/model/messages" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/utils/logging" -) - -// Engine represents the provider engine, used to spread block proposals across -// the flow system, to non-consensus nodes. It makes sense to use a separate -// engine to isolate the consensus algorithm from other processes, which allows -// to create a different underlying protocol for consensus nodes, which have a -// higher priority to receive block proposals, and other nodes -type Engine struct { - unit *engine.Unit // used for concurrency & shutdown - log zerolog.Logger // used to log relevant actions with context - message module.EngineMetrics // used to track sent & received messages - tracer module.Tracer - con network.Conduit // used to talk to other nodes on the network - state protocol.State // used to access the protocol state - me module.Local // used to access local node information -} - -// New creates a new block provider engine. -func New( - log zerolog.Logger, - message module.EngineMetrics, - tracer module.Tracer, - net network.Network, - state protocol.State, - me module.Local, -) (*Engine, error) { - - // initialize the propagation engine with its dependencies - e := &Engine{ - unit: engine.NewUnit(), - log: log.With().Str("engine", "provider").Logger(), - message: message, - tracer: tracer, - state: state, - me: me, - } - - // register the engine with the network layer and store the conduit - con, err := net.Register(channels.PushBlocks, e) - if err != nil { - return nil, fmt.Errorf("could not register engine: %w", err) - } - - e.con = con - - return e, nil -} - -// Ready returns a ready channel that is closed once the engine has fully -// started. For the provider engine, we consider the engine up and running -// upon initialization. -func (e *Engine) Ready() <-chan struct{} { - return e.unit.Ready() -} - -// Done returns a done channel that is closed once the engine has fully stopped. -// For the ingestion engine, it only waits for all submit goroutines to end. -func (e *Engine) Done() <-chan struct{} { - return e.unit.Done() -} - -// SubmitLocal submits an event originating on the local node. -func (e *Engine) SubmitLocal(event interface{}) { - e.unit.Launch(func() { - err := e.ProcessLocal(event) - if err != nil { - engine.LogError(e.log, err) - } - }) -} - -// Submit submits the given event from the node with the given origin ID -// for processing in a non-blocking manner. It returns instantly and logs -// a potential processing error internally when done. -func (e *Engine) Submit(channel channels.Channel, originID flow.Identifier, event interface{}) { - e.unit.Launch(func() { - err := e.Process(channel, originID, event) - if err != nil { - engine.LogError(e.log, err) - } - }) -} - -// ProcessLocal processes an event originating on the local node. -func (e *Engine) ProcessLocal(event interface{}) error { - return e.unit.Do(func() error { - return e.process(e.me.NodeID(), event) - }) -} - -// Process processes the given event from the node with the given origin ID in -// a blocking manner. It returns the potential processing error when done. -func (e *Engine) Process(channel channels.Channel, originID flow.Identifier, event interface{}) error { - return e.unit.Do(func() error { - return e.process(originID, event) - }) -} - -// process processes the given ingestion engine event. Events that are given -// to this function originate within the provider engine on the node with the -// given origin ID. -func (e *Engine) process(originID flow.Identifier, event interface{}) error { - switch ev := event.(type) { - case *messages.BlockProposal: - return e.onBlockProposal(originID, ev) - default: - return fmt.Errorf("invalid event type (%T)", event) - } -} - -// onBlockProposal is used when we want to broadcast a local block to the network. -func (e *Engine) onBlockProposal(originID flow.Identifier, proposal *messages.BlockProposal) error { - block := proposal.Block.ToInternal() - log := e.log.With(). - Hex("origin_id", originID[:]). - Uint64("block_view", block.Header.View). - Hex("block_id", logging.Entity(block.Header)). - Hex("parent_id", block.Header.ParentID[:]). - Hex("signer", block.Header.ProposerID[:]). - Logger() - - log.Info().Msg("block proposal submitted for propagation") - - // currently, only accept blocks that come from our local consensus - localID := e.me.NodeID() - if originID != localID { - return engine.NewInvalidInputErrorf("non-local block (nodeID: %x)", originID) - } - - // determine the nodes we should send the block to - recipients, err := e.state.Final().Identities(filter.And( - filter.Not(filter.Ejected), - filter.Not(filter.HasRole(flow.RoleConsensus)), - )) - if err != nil { - return fmt.Errorf("could not get recipients: %w", err) - } - - // submit the block to the targets - err = e.con.Publish(proposal, recipients.NodeIDs()...) - if err != nil { - return fmt.Errorf("could not broadcast block: %w", err) - } - - e.message.MessageSent(metrics.EngineConsensusProvider, metrics.MessageBlockProposal) - - log.Info().Msg("block proposal propagated to non-consensus nodes") - - return nil -} diff --git a/engine/consensus/provider/engine_test.go b/engine/consensus/provider/engine_test.go deleted file mode 100644 index 7c9a1f56f60..00000000000 --- a/engine/consensus/provider/engine_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - -package provider - -import ( - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/metrics" - module "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/network/mocknetwork" - protocol "github.com/onflow/flow-go/state/protocol/mock" - "github.com/onflow/flow-go/utils/unittest" -) - -type Suite struct { - suite.Suite - - me *module.Local - conduit *mocknetwork.Conduit - state *protocol.State - final *protocol.Snapshot - - identities flow.IdentityList - - engine *Engine -} - -func TestProviderEngine(t *testing.T) { - suite.Run(t, new(Suite)) -} - -func (suite *Suite) SetupTest() { - - suite.me = new(module.Local) - suite.conduit = new(mocknetwork.Conduit) - suite.state = new(protocol.State) - suite.final = new(protocol.Snapshot) - - suite.engine = &Engine{ - me: suite.me, - state: suite.state, - con: suite.conduit, - message: metrics.NewNoopCollector(), - tracer: trace.NewNoopTracer(), - } - - suite.identities = unittest.CompleteIdentitySet() - localID := suite.identities[0].NodeID - - suite.me.On("NodeID").Return(localID) - suite.state.On("Final").Return(suite.final) - suite.final.On("Identities", mock.Anything).Return( - func(f flow.IdentityFilter) flow.IdentityList { return suite.identities.Filter(f) }, - func(flow.IdentityFilter) error { return nil }, - ) -} - -// proposals submitted by remote nodes should not be accepted. -func (suite *Suite) TestOnBlockProposal_RemoteOrigin() { - - proposal := unittest.ProposalFixture() - // message submitted by remote node - err := suite.engine.onBlockProposal(suite.identities[1].NodeID, proposal) - suite.Assert().Error(err) -} - -func (suite *Suite) OnBlockProposal_Success() { - - proposal := unittest.ProposalFixture() - - params := []interface{}{proposal} - for _, identity := range suite.identities { - // skip consensus nodes - if identity.Role == flow.RoleConsensus { - continue - } - params = append(params, identity.NodeID) - } - - suite.conduit.On("Publish", params...).Return(nil).Once() - - err := suite.engine.onBlockProposal(suite.me.NodeID(), proposal) - suite.Require().Nil(err) - suite.conduit.AssertExpectations(suite.T()) -} diff --git a/engine/consensus/sealing/core.go b/engine/consensus/sealing/core.go index 7d36760f86a..942d489e971 100644 --- a/engine/consensus/sealing/core.go +++ b/engine/consensus/sealing/core.go @@ -197,7 +197,7 @@ func (c *Core) RepopulateAssignmentCollectorTree(payloads storage.Payloads) erro // at this point we have processed all results in range (lastSealedBlock, lastFinalizedBlock]. // Now, we add all known results for any valid block that descends from the latest finalized block: - validPending, err := finalizedSnapshot.ValidDescendants() + validPending, err := finalizedSnapshot.Descendants() if err != nil { return fmt.Errorf("could not retrieve valid pending blocks from finalized snapshot: %w", err) } @@ -585,12 +585,12 @@ func (c *Core) prune(parentSpan otelTrace.Span, finalized, lastSealed *flow.Head } err = c.requestTracker.PruneUpToHeight(lastSealed.Height) - if err != nil && !mempool.IsDecreasingPruningHeightError(err) { + if err != nil && !mempool.IsBelowPrunedThresholdError(err) { return fmt.Errorf("could not request tracker at block up to height %d: %w", lastSealed.Height, err) } err = c.sealsMempool.PruneUpToHeight(lastSealed.Height) // prune candidate seals mempool - if err != nil && !mempool.IsDecreasingPruningHeightError(err) { + if err != nil && !mempool.IsBelowPrunedThresholdError(err) { return fmt.Errorf("could not prune seals mempool at block up to height %d: %w", lastSealed.Height, err) } diff --git a/engine/consensus/sealing/core_test.go b/engine/consensus/sealing/core_test.go index 36899ee86c2..4dfbc31d50c 100644 --- a/engine/consensus/sealing/core_test.go +++ b/engine/consensus/sealing/core_test.go @@ -742,9 +742,9 @@ func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree( } } - // ValidDescendants has to return all valid descendants from finalized block + // Descendants has to return all valid descendants from finalized block finalSnapShot := unittest.StateSnapshotForKnownBlock(s.IncorporatedBlock, nil) - finalSnapShot.On("ValidDescendants").Return(blockChildren, nil) + finalSnapShot.On("Descendants").Return(blockChildren, nil) s.State.On("Final").Return(finalSnapShot) core, err := NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), @@ -809,7 +809,7 @@ func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree_ finalSnapShot := unittest.StateSnapshotForKnownBlock(s.rootHeader, nil) s.Snapshots[s.rootHeader.ID()] = finalSnapShot // root snapshot has no pending children - finalSnapShot.On("ValidDescendants").Return(nil, nil) + finalSnapShot.On("Descendants").Return(nil, nil) // set up sealing segment finalSnapShot.On("SealingSegment").Return( &flow.SealingSegment{ diff --git a/engine/consensus/sealing/engine.go b/engine/consensus/sealing/engine.go index a9cad3ac4ef..ae432725bd6 100644 --- a/engine/consensus/sealing/engine.go +++ b/engine/consensus/sealing/engine.go @@ -432,7 +432,7 @@ func (e *Engine) Done() <-chan struct{} { } // OnFinalizedBlock implements the `OnFinalizedBlock` callback from the `hotstuff.FinalizationConsumer` -// (1) Informs sealing.Core about finalization of respective block. +// It informs sealing.Core about finalization of respective block. // // CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages // from external nodes cannot be considered as inputs to this function @@ -441,7 +441,7 @@ func (e *Engine) OnFinalizedBlock(*model.Block) { } // OnBlockIncorporated implements `OnBlockIncorporated` from the `hotstuff.FinalizationConsumer` -// (1) Processes all execution results that were incorporated in parent block payload. +// It processes all execution results that were incorporated in parent block payload. // // CAUTION: the input to this callback is treated as trusted; precautions should be taken that messages // from external nodes cannot be considered as inputs to this function diff --git a/engine/consensus/sealing/engine_test.go b/engine/consensus/sealing/engine_test.go index cfbf43b3c64..e5adb345460 100644 --- a/engine/consensus/sealing/engine_test.go +++ b/engine/consensus/sealing/engine_test.go @@ -89,7 +89,7 @@ func (s *SealingEngineSuite) TestOnFinalizedBlock() { s.state.On("Final").Return(unittest.StateSnapshotForKnownBlock(finalizedBlock, nil)) s.core.On("ProcessFinalizedBlock", finalizedBlockID).Return(nil).Once() - s.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock, finalizedBlock.View-1)) + s.engine.OnFinalizedBlock(model.BlockFromFlow(finalizedBlock)) // matching engine has at least 100ms ticks for processing events time.Sleep(1 * time.Second) @@ -121,7 +121,7 @@ func (s *SealingEngineSuite) TestOnBlockIncorporated() { headers.On("ByBlockID", incorporatedBlockID).Return(incorporatedBlock, nil).Once() s.engine.headers = headers - s.engine.OnBlockIncorporated(model.BlockFromFlow(incorporatedBlock, incorporatedBlock.View-1)) + s.engine.OnBlockIncorporated(model.BlockFromFlow(incorporatedBlock)) // matching engine has at least 100ms ticks for processing events time.Sleep(1 * time.Second) diff --git a/engine/enqueue.go b/engine/enqueue.go index ab3aede7b51..7e4d645f24c 100644 --- a/engine/enqueue.go +++ b/engine/enqueue.go @@ -14,10 +14,17 @@ type Message struct { Payload interface{} } -// MessageStore is the interface to abstract how messages are buffered in memory before -// being handled by the engine +// MessageStore is the interface to abstract how messages are buffered in memory +// while waiting to be processed. type MessageStore interface { + // Put adds the message to the message store. It returns true if the message + // is stored, and false if it is immediately dropped. + // Note: depending on the implementation, message stores might drop messages + // later according to their internal ejection policy. In other words, a return + // value of `true` does _not imply_ that the message is eventually processed. Put(*Message) bool + // Get retrieves the next message from the message store. It returns true if + // a message is retrieved, and false if the message store is empty. Get() (*Message, bool) } diff --git a/engine/execution/computation/committer/committer.go b/engine/execution/computation/committer/committer.go index 76b1be13399..d998e4b214e 100644 --- a/engine/execution/computation/committer/committer.go +++ b/engine/execution/computation/committer/committer.go @@ -49,7 +49,7 @@ func (s *LedgerViewCommitter) commitView(view state.View, baseState flow.StateCo func (s *LedgerViewCommitter) collectProofs(view state.View, baseState flow.StateCommitment) (proof []byte, err error) { // get all deduplicated register IDs - allIds := view.AllRegisters() + allIds := view.AllRegisterIDs() keys := make([]ledger.Key, 0, len(allIds)) for _, id := range allIds { keys = append(keys, execState.RegisterIDToKey(id)) diff --git a/engine/execution/computation/committer/committer_test.go b/engine/execution/computation/committer/committer_test.go index 894c6281ef2..58791836320 100644 --- a/engine/execution/computation/committer/committer_test.go +++ b/engine/execution/computation/committer/committer_test.go @@ -36,8 +36,7 @@ func TestLedgerViewCommitter(t *testing.T) { view := fvmUtils.NewSimpleView() err := view.Set( - "owner", - "key", + flow.NewRegisterID("owner", "key"), []byte{1}, ) require.NoError(t, err) diff --git a/engine/execution/computation/computer/computer.go b/engine/execution/computation/computer/computer.go index b2a0dafd0aa..a208dcab0d1 100644 --- a/engine/execution/computation/computer/computer.go +++ b/engine/execution/computation/computer/computer.go @@ -31,6 +31,7 @@ const ( ) type collectionItem struct { + blockId flow.Identifier blockIdStr string collectionIndex int @@ -193,8 +194,6 @@ func (e *blockComputer) ExecuteBlock( return nil, fmt.Errorf("failed to execute transactions: %w", err) } - // TODO: compute block fees & reward payments - return results, nil } @@ -222,6 +221,7 @@ func (e *blockComputer) getRootSpanAndCollections( collections = append( collections, collectionItem{ + blockId: blockId, blockIdStr: blockIdStr, collectionIndex: idx, CompleteCollection: collection, @@ -255,6 +255,7 @@ func (e *blockComputer) getRootSpanAndCollections( collections = append( collections, collectionItem{ + blockId: blockId, blockIdStr: blockIdStr, collectionIndex: len(collections), CompleteCollection: &entity.CompleteCollection{ @@ -353,6 +354,8 @@ func (e *blockComputer) executeBlock( Hex("block_id", logging.Entity(block)). Msg("all views committed") + e.metrics.ExecutionBlockCachedPrograms(derivedBlockData.CachedPrograms()) + executionDataID, err := e.executionDataProvider.Provide( ctx, block.Height(), @@ -430,11 +433,11 @@ func (e *blockComputer) executeCollection( Int64("time_spent_in_ms", time.Since(startedAt).Milliseconds()). Msg("collection executed") - stats := collector.CommitCollection( + collector.CommitCollection( collection, + startedAt, collectionView) - e.metrics.ExecutionCollectionExecuted(time.Since(startedAt), stats) return startTxIndex + uint32(len(txns)), nil } diff --git a/engine/execution/computation/computer/computer_test.go b/engine/execution/computation/computer/computer_test.go index 9ebaec7f598..534991f0696 100644 --- a/engine/execution/computation/computer/computer_test.go +++ b/engine/execution/computation/computer/computer_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/onflow/cadence" + "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" @@ -34,6 +35,8 @@ import ( reusableRuntime "github.com/onflow/flow-go/fvm/runtime" "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/systemcontracts" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/convert/fixtures" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/epochs" @@ -48,6 +51,31 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +func incStateCommitment(startState flow.StateCommitment) flow.StateCommitment { + endState := flow.StateCommitment(startState) + endState[0] += 1 + return endState +} + +type fakeCommitter struct { + callCount int +} + +func (committer *fakeCommitter) CommitView( + view state.View, + startState flow.StateCommitment, +) ( + flow.StateCommitment, + []byte, + *ledger.TrieUpdate, + error, +) { + committer.callCount++ + + endState := incStateCommitment(startState) + return endState, []byte{byte(committer.callCount)}, nil, nil +} + func TestBlockExecutor_ExecuteBlock(t *testing.T) { rag := &RandomAddressGenerator{} @@ -58,23 +86,26 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { t.Run("single collection", func(t *testing.T) { - execCtx := fvm.NewContext() + execCtx := fvm.NewContext( + fvm.WithDerivedBlockData(derived.NewEmptyDerivedBlockData()), + ) vm := new(computermock.VirtualMachine) vm.On("Run", mock.Anything, mock.Anything, mock.Anything). Return(nil). Run(func(args mock.Arguments) { - // ctx := args[0].(fvm.Context) + ctx := args[0].(fvm.Context) tx := args[1].(*fvm.TransactionProcedure) tx.Events = generateEvents(1, tx.TxIndex) + + getSetAProgram(t, ctx.DerivedBlockData) }). Times(2 + 1) // 2 txs in collection + system chunk - committer := new(computermock.ViewCommitter) - committer.On("CommitView", mock.Anything, mock.Anything). - Return(nil, nil, nil, nil). - Times(2 + 1) // 2 txs in collection + system chunk + committer := &fakeCommitter{ + callCount: 0, + } exemetrics := new(modulemock.ExecutionMetrics) exemetrics.On("ExecutionCollectionExecuted", @@ -94,6 +125,20 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { Return(nil). Times(2 + 1) // 2 txs in collection + system chunk tx + exemetrics.On( + "ExecutionChunkDataPackGenerated", + mock.Anything, + mock.Anything). + Return(nil). + Times(2) // 1 collection + system collection + + expectedProgramsInCache := 1 // we set one program in the cache + exemetrics.On( + "ExecutionBlockCachedPrograms", + expectedProgramsInCache). + Return(nil). + Times(1) // 1 block + bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))) trackerStorage := mocktracker.NewMockStorage() @@ -119,9 +164,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { // create a block with 1 collection with 2 transactions block := generateBlock(1, 2, rag) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) result, err := exe.ExecuteBlock( context.Background(), @@ -131,6 +174,58 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { assert.NoError(t, err) assert.Len(t, result.StateSnapshots, 1+1) // +1 system chunk assert.Len(t, result.TrieUpdates, 1+1) // +1 system chunk + assert.Len(t, result.Chunks, 1+1) // +1 system chunk + assert.Len(t, result.ChunkDataPacks, 1+1) // +1 system chunk + + require.Equal(t, 2, committer.callCount) + + // regular collection chunk + chunk1 := result.Chunks[0] + + assert.Equal(t, block.ID(), chunk1.BlockID) + assert.Equal(t, uint(0), chunk1.CollectionIndex) + assert.Equal(t, uint64(2), chunk1.NumberOfTransactions) + assert.Equal(t, result.EventsHashes[0], chunk1.EventCollection) + + assert.Equal(t, *block.StartState, chunk1.StartState) + + expectedChunk1EndState := incStateCommitment(*block.StartState) + + assert.NotEqual(t, *block.StartState, chunk1.EndState) + assert.NotEqual(t, flow.DummyStateCommitment, chunk1.EndState) + assert.Equal(t, expectedChunk1EndState, chunk1.EndState) + + chunkDataPack1 := result.ChunkDataPacks[0] + + assert.Equal(t, chunk1.ID(), chunkDataPack1.ChunkID) + assert.Equal(t, *block.StartState, chunkDataPack1.StartState) + assert.Equal(t, []byte{1}, chunkDataPack1.Proof) + assert.NotNil(t, chunkDataPack1.Collection) + + // system chunk is special case, but currently also 1 tx + chunk2 := result.Chunks[1] + assert.Equal(t, block.ID(), chunk2.BlockID) + assert.Equal(t, uint(1), chunk2.CollectionIndex) + assert.Equal(t, uint64(1), chunk2.NumberOfTransactions) + assert.Equal(t, result.EventsHashes[1], chunk2.EventCollection) + + assert.Equal(t, expectedChunk1EndState, chunk2.StartState) + + expectedChunk2EndState := incStateCommitment(expectedChunk1EndState) + + assert.NotEqual(t, *block.StartState, chunk2.EndState) + assert.NotEqual(t, flow.DummyStateCommitment, chunk2.EndState) + assert.NotEqual(t, expectedChunk1EndState, chunk2.EndState) + assert.Equal(t, expectedChunk2EndState, chunk2.EndState) + + chunkDataPack2 := result.ChunkDataPacks[1] + + assert.Equal(t, chunk2.ID(), chunkDataPack2.ChunkID) + assert.Equal(t, chunk2.StartState, chunkDataPack2.StartState) + assert.Equal(t, []byte{2}, chunkDataPack2.Proof) + assert.Nil(t, chunkDataPack2.Collection) + + assert.Equal(t, expectedChunk2EndState, result.EndState) assertEventHashesMatch(t, 1+1, result) @@ -178,9 +273,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { Return(nil, nil, nil, nil). Once() // just system chunk - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) result, err := exe.ExecuteBlock(context.Background(), block, view, derivedBlockData) assert.NoError(t, err) @@ -222,9 +315,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { opts := append(baseOpts, contextOptions...) ctx := fvm.NewContext(opts...) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) baseBootstrapOpts := []fvm.BootstrapProcedureOption{ fvm.WithInitialTokenSupply(unittest.GenesisTokenSupply), @@ -316,7 +407,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { Run(func(args mock.Arguments) { tx := args[1].(*fvm.TransactionProcedure) - tx.Err = fvmErrors.NewInvalidAddressErrorf(flow.Address{}, "no payer address provided") + tx.Err = fvmErrors.NewInvalidAddressErrorf( + flow.EmptyAddress, + "no payer address provided") // create dummy events tx.Events = generateEvents(eventsPerTransaction, tx.TxIndex) }). @@ -327,9 +420,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { Return(nil, nil, nil, nil). Times(collectionCount + 1) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) result, err := exe.ExecuteBlock(context.Background(), block, view, derivedBlockData) assert.NoError(t, err) @@ -366,7 +457,9 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { for _, t := range c.Transactions { txResult := flow.TransactionResult{ TransactionID: t.ID(), - ErrorMessage: fvmErrors.NewInvalidAddressErrorf(flow.Address{}, "no payer address provided").Error(), + ErrorMessage: fvmErrors.NewInvalidAddressErrorf( + flow.EmptyAddress, + "no payer address provided").Error(), } expectedResults = append(expectedResults, txResult) } @@ -403,22 +496,27 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { serviceEvents, err := systemcontracts.ServiceEventsForChain(execCtx.Chain.ChainID()) require.NoError(t, err) - serviceEventA := cadence.Event{ - EventType: &cadence.EventType{ - Location: common.AddressLocation{ - Address: common.Address(serviceEvents.EpochSetup.Address), - }, - QualifiedIdentifier: serviceEvents.EpochSetup.QualifiedIdentifier(), - }, + payload, err := json.Decode(nil, []byte(fixtures.EpochSetupFixtureJSON)) + require.NoError(t, err) + + serviceEventA, ok := payload.(cadence.Event) + require.True(t, ok) + + serviceEventA.EventType.Location = common.AddressLocation{ + Address: common.Address(serviceEvents.EpochSetup.Address), } - serviceEventB := cadence.Event{ - EventType: &cadence.EventType{ - Location: common.AddressLocation{ - Address: common.Address(serviceEvents.EpochCommit.Address), - }, - QualifiedIdentifier: serviceEvents.EpochCommit.QualifiedIdentifier(), - }, + serviceEventA.EventType.QualifiedIdentifier = serviceEvents.EpochSetup.QualifiedIdentifier() + + payload, err = json.Decode(nil, []byte(fixtures.EpochCommitFixtureJSON)) + require.NoError(t, err) + + serviceEventB, ok := payload.(cadence.Event) + require.True(t, ok) + + serviceEventB.EventType.Location = common.AddressLocation{ + Address: common.Address(serviceEvents.EpochCommit.Address), } + serviceEventB.EventType.QualifiedIdentifier = serviceEvents.EpochCommit.QualifiedIdentifier() // events to emit for each iteration/transaction events := make([][]cadence.Event, totalTransactionCount) @@ -477,9 +575,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { prov) require.NoError(t, err) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) result, err := exe.ExecuteBlock(context.Background(), block, view, derived.NewEmptyDerivedBlockData()) require.NoError(t, err) @@ -568,11 +664,11 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { const transactionCount = 2 block := generateBlock(collectionCount, transactionCount, rag) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) - err = view.Set(string(address.Bytes()), state.AccountStatusKey, environment.NewAccountStatus().ToBytes()) + err = view.Set( + flow.AccountStatusRegisterID(flow.BytesToAddress(address.Bytes())), + environment.NewAccountStatus().ToBytes()) require.NoError(t, err) result, err := exe.ExecuteBlock(context.Background(), block, view, derived.NewEmptyDerivedBlockData()) @@ -666,11 +762,11 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { block := generateBlock(collectionCount, transactionCount, rag) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + view := delta.NewDeltaView(nil) - err = view.Set(string(address.Bytes()), state.AccountStatusKey, environment.NewAccountStatus().ToBytes()) + err = view.Set( + flow.AccountStatusRegisterID(flow.BytesToAddress(address.Bytes())), + environment.NewAccountStatus().ToBytes()) require.NoError(t, err) result, err := exe.ExecuteBlock(context.Background(), block, view, derived.NewEmptyDerivedBlockData()) @@ -843,9 +939,7 @@ func Test_AccountStatusRegistersAreIncluded(t *testing.T) { key, err := unittest.AccountKeyDefaultFixture() require.NoError(t, err) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return ledger.Get(owner, key) - }) + view := delta.NewDeltaView(ledger) txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) @@ -892,10 +986,7 @@ func Test_AccountStatusRegistersAreIncluded(t *testing.T) { registerTouches := view.Interactions().RegisterTouches() // make sure check for account status has been registered - id := flow.RegisterID{ - Owner: string(address.Bytes()), - Key: state.AccountStatusKey, - } + id := flow.AccountStatusRegisterID(address) require.Contains(t, registerTouches, id) } @@ -920,9 +1011,12 @@ func Test_ExecutingSystemCollection(t *testing.T) { noopCollector := metrics.NewNoopCollector() - metrics := new(modulemock.ExecutionMetrics) expectedNumberOfEvents := 2 - expectedEventSize := 912 + expectedEventSize := 911 + // bootstrapping does not cache programs + expectedCachedPrograms := 0 + + metrics := new(modulemock.ExecutionMetrics) metrics.On("ExecutionCollectionExecuted", mock.Anything, // duration mock.Anything). // stats @@ -940,6 +1034,19 @@ func Test_ExecutingSystemCollection(t *testing.T) { Return(nil). Times(1) // system chunk tx + metrics.On( + "ExecutionChunkDataPackGenerated", + mock.Anything, + mock.Anything). + Return(nil). + Times(1) // system collection + + metrics.On( + "ExecutionBlockCachedPrograms", + expectedCachedPrograms). + Return(nil). + Times(1) // block + bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore()))) trackerStorage := mocktracker.NewMockStorage() @@ -969,7 +1076,7 @@ func Test_ExecutingSystemCollection(t *testing.T) { // create empty block, it will have system collection attached while executing block := generateBlock(0, 0, rag) - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) result, err := exe.ExecuteBlock(context.Background(), block, view, derived.NewEmptyDerivedBlockData()) assert.NoError(t, err) @@ -988,8 +1095,8 @@ func Test_ExecutingSystemCollection(t *testing.T) { module.ExecutionResultStats{ EventCounts: expectedNumberOfEvents, EventSize: expectedEventSize, - NumberOfRegistersTouched: 50, - NumberOfBytesWrittenToRegisters: 3404, + NumberOfRegistersTouched: 66, + NumberOfBytesWrittenToRegisters: 4214, NumberOfCollections: 1, NumberOfTransactions: 1, }, @@ -1069,3 +1176,30 @@ func generateEvents(eventCount int, txIndex uint32) []flow.Event { } return events } + +func getSetAProgram(t *testing.T, derivedBlockData *derived.DerivedBlockData) { + + derivedTxnData, err := derivedBlockData.NewDerivedTransactionData( + 0, + 0) + require.NoError(t, err) + + loc := common.AddressLocation{ + Name: "SomeContract", + Address: common.MustBytesToAddress([]byte{0x1}), + } + _, _, got := derivedTxnData.GetProgram( + loc, + ) + if got { + return + } + + derivedTxnData.SetProgram( + loc, + &derived.Program{}, + &state.State{}, + ) + err = derivedTxnData.Commit() + require.NoError(t, err) +} diff --git a/engine/execution/computation/computer/result_collector.go b/engine/execution/computation/computer/result_collector.go index d2f9dfd0a4a..3112239df01 100644 --- a/engine/execution/computation/computer/result_collector.go +++ b/engine/execution/computation/computer/result_collector.go @@ -3,6 +3,7 @@ package computer import ( "fmt" "sync" + "time" "github.com/hashicorp/go-multierror" otelTrace "go.opentelemetry.io/otel/trace" @@ -33,6 +34,12 @@ type ViewCommitter interface { ) } +type collectionResult struct { + collectionItem + startTime time.Time + state.View +} + type resultCollector struct { tracer module.Tracer blockSpan otelTrace.Span @@ -42,20 +49,15 @@ type resultCollector struct { closeOnce sync.Once committer ViewCommitter - state flow.StateCommitment - committerInputChan chan state.View + committerInputChan chan collectionResult committerDoneChan chan struct{} committerError error - eventHasherInputChan chan flow.EventsList - eventHasherDoneChan chan struct{} - eventHasherError error - - signer module.Local - spockHasher hash.Hasher - spockHasherInputChan chan *delta.SpockSnapshot - spockHasherDoneChan chan struct{} - spockHasherError error + signer module.Local + spockHasher hash.Hasher + snapshotHasherInputChan chan collectionResult + snapshotHasherDoneChan chan struct{} + snapshotHasherError error result *execution.ComputationResult } @@ -71,25 +73,21 @@ func newResultCollector( numCollections int, ) *resultCollector { collector := &resultCollector{ - tracer: tracer, - blockSpan: blockSpan, - metrics: metrics, - committer: committer, - state: *block.StartState, - committerInputChan: make(chan state.View, numCollections), - committerDoneChan: make(chan struct{}), - eventHasherInputChan: make(chan flow.EventsList, numCollections), - eventHasherDoneChan: make(chan struct{}), - signer: signer, - spockHasher: spockHasher, - spockHasherInputChan: make(chan *delta.SpockSnapshot, numCollections), - spockHasherDoneChan: make(chan struct{}), - result: execution.NewEmptyComputationResult(block), + tracer: tracer, + blockSpan: blockSpan, + metrics: metrics, + committer: committer, + committerInputChan: make(chan collectionResult, numCollections), + committerDoneChan: make(chan struct{}), + signer: signer, + spockHasher: spockHasher, + snapshotHasherInputChan: make(chan collectionResult, numCollections), + snapshotHasherDoneChan: make(chan struct{}), + result: execution.NewEmptyComputationResult(block), } go collector.runCollectionCommitter() - go collector.runEventsHasher() - go collector.runSpockHasher() + go collector.runSnapshotHasher() return collector } @@ -97,67 +95,95 @@ func newResultCollector( func (collector *resultCollector) runCollectionCommitter() { defer close(collector.committerDoneChan) - for view := range collector.committerInputChan { + for collection := range collector.committerInputChan { span := collector.tracer.StartSpanFromParent( collector.blockSpan, trace.EXECommitDelta) - stateCommit, proof, trieUpdate, err := collector.committer.CommitView( - view, - collector.state) + startState := collector.result.EndState + endState, proof, trieUpdate, err := collector.committer.CommitView( + collection.View, + startState) if err != nil { - collector.committerError = fmt.Errorf("committer failed: %w", err) + collector.committerError = fmt.Errorf( + "commit view failed: %w", + err) return } collector.result.StateCommitments = append( collector.result.StateCommitments, - stateCommit) + endState) collector.result.Proofs = append(collector.result.Proofs, proof) collector.result.TrieUpdates = append( collector.result.TrieUpdates, trieUpdate) - collector.state = stateCommit - span.End() - } -} - -func (collector *resultCollector) runEventsHasher() { - defer close(collector.eventHasherDoneChan) - - for data := range collector.eventHasherInputChan { - span := collector.tracer.StartSpanFromParent( - collector.blockSpan, - trace.EXEHashEvents) - - rootHash, err := flow.EventsMerkleRootHash(data) + eventsHash, err := flow.EventsMerkleRootHash( + collector.result.Events[collection.collectionIndex]) if err != nil { - collector.eventHasherError = fmt.Errorf( - "event hasher failed: %w", + collector.committerError = fmt.Errorf( + "hash events failed: %w", err) return } collector.result.EventsHashes = append( collector.result.EventsHashes, - rootHash) + eventsHash) + + chunk := flow.NewChunk( + collection.blockId, + collection.collectionIndex, + startState, + len(collection.transactions), + eventsHash, + endState) + collector.result.Chunks = append(collector.result.Chunks, chunk) + + var flowCollection *flow.Collection + if !collection.isSystemCollection { + collectionStruct := collection.CompleteCollection.Collection() + flowCollection = &collectionStruct + } + + collector.result.ChunkDataPacks = append( + collector.result.ChunkDataPacks, + flow.NewChunkDataPack( + chunk.ID(), + startState, + proof, + flowCollection)) + + collector.metrics.ExecutionChunkDataPackGenerated( + len(proof), + len(collection.transactions)) + + collector.result.EndState = endState span.End() } } -func (collector *resultCollector) runSpockHasher() { - defer close(collector.spockHasherDoneChan) +func (collector *resultCollector) runSnapshotHasher() { + defer close(collector.snapshotHasherDoneChan) + + for collection := range collector.snapshotHasherInputChan { + + snapshot := collection.View.(*delta.View).Interactions() + collector.result.AddCollection(snapshot) + + collector.metrics.ExecutionCollectionExecuted( + time.Since(collection.startTime), + collector.result.CollectionStats(collection.collectionIndex)) - for snapshot := range collector.spockHasherInputChan { spock, err := collector.signer.SignFunc( snapshot.SpockSecret, collector.spockHasher, SPOCKProve) if err != nil { - collector.spockHasherError = fmt.Errorf( - "spock hasher failed: %w", + collector.snapshotHasherError = fmt.Errorf( + "signing spock hash failed: %w", err) return } @@ -177,43 +203,40 @@ func (collector *resultCollector) AddTransactionResult( func (collector *resultCollector) CommitCollection( collection collectionItem, + startTime time.Time, collectionView state.View, -) module.ExecutionResultStats { +) { - select { - case collector.committerInputChan <- collectionView: - // Do nothing - case <-collector.committerDoneChan: - // Committer exited (probably due to an error) + result := collectionResult{ + collectionItem: collection, + startTime: startTime, + View: collectionView, } select { - case collector.eventHasherInputChan <- collector.result.Events[collection.collectionIndex]: + case collector.committerInputChan <- result: // Do nothing - case <-collector.eventHasherDoneChan: - // Events hasher exited (probably due to an error) + case <-collector.committerDoneChan: + // Committer exited (probably due to an error) } - snapshot := collectionView.(*delta.View).Interactions() select { - case collector.spockHasherInputChan <- snapshot: + case collector.snapshotHasherInputChan <- result: // do nothing - case <-collector.spockHasherDoneChan: - // Spock hasher exited (probably due to an error) + case <-collector.snapshotHasherDoneChan: + // Snapshot hasher exited (probably due to an error) } - - collector.result.AddCollection(snapshot) - return collector.result.CollectionStats(collection.collectionIndex) } func (collector *resultCollector) Stop() { collector.closeOnce.Do(func() { close(collector.committerInputChan) - close(collector.eventHasherInputChan) - close(collector.spockHasherInputChan) + close(collector.snapshotHasherInputChan) }) } +// TODO(patrick): refactor execution receipt generation from ingress engine +// to here to improve benchmarking. func (collector *resultCollector) Finalize() ( *execution.ComputationResult, error, @@ -221,20 +244,15 @@ func (collector *resultCollector) Finalize() ( collector.Stop() <-collector.committerDoneChan - <-collector.eventHasherDoneChan - <-collector.spockHasherDoneChan + <-collector.snapshotHasherDoneChan var err error if collector.committerError != nil { err = multierror.Append(err, collector.committerError) } - if collector.eventHasherError != nil { - err = multierror.Append(err, collector.eventHasherError) - } - - if collector.spockHasherError != nil { - err = multierror.Append(err, collector.spockHasherError) + if collector.snapshotHasherError != nil { + err = multierror.Append(err, collector.snapshotHasherError) } if err != nil { diff --git a/engine/execution/computation/execution_verification_test.go b/engine/execution/computation/execution_verification_test.go index 26c9fa225d4..39f0ffbb5da 100644 --- a/engine/execution/computation/execution_verification_test.go +++ b/engine/execution/computation/execution_verification_test.go @@ -705,7 +705,10 @@ func executeBlockAndVerifyWithParameters(t *testing.T, prov) require.NoError(t, err) - view := delta.NewView(state.LedgerGetRegister(ledger, initialCommit)) + view := delta.NewDeltaView( + state.NewLedgerStorageSnapshot( + ledger, + initialCommit)) executableBlock := unittest.ExecutableBlockFromTransactions(chain.ChainID(), txs) executableBlock.StartState = &initialCommit @@ -730,8 +733,13 @@ func executeBlockAndVerifyWithParameters(t *testing.T, prevResultId := unittest.IdentifierFixture() - _, chdps, er, err := execution.GenerateExecutionResultAndChunkDataPacks(metrics.NewNoopCollector(), prevResultId, initialCommit, computationResult) - require.NoError(t, err) + chdps := computationResult.ChunkDataPacks + er := flow.NewExecutionResult( + prevResultId, + executableBlock.ID(), + computationResult.Chunks, + computationResult.ConvertedServiceEvents, + computationResult.ExecutionDataID) verifier := chunks.NewChunkVerifier(vm, fvmContext, logger) diff --git a/engine/execution/computation/manager_benchmark_test.go b/engine/execution/computation/manager_benchmark_test.go index ec439fd035f..46b998768de 100644 --- a/engine/execution/computation/manager_benchmark_test.go +++ b/engine/execution/computation/manager_benchmark_test.go @@ -166,7 +166,7 @@ func BenchmarkComputeBlock(b *testing.B) { derivedChainData: derivedChainData, } - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) blockView := view.NewChild() b.SetParallelism(1) diff --git a/engine/execution/computation/manager_test.go b/engine/execution/computation/manager_test.go index b92fcb36f16..14d4402e678 100644 --- a/engine/execution/computation/manager_test.go +++ b/engine/execution/computation/manager_test.go @@ -149,7 +149,7 @@ func TestComputeBlockWithStorage(t *testing.T) { tracer: trace.NewNoopTracer(), } - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) blockView := view.NewChild() returnedComputationResult, err := engine.ComputeBlock(context.Background(), executableBlock, blockView) @@ -201,7 +201,10 @@ func TestComputeBlock_Uploader(t *testing.T) { tracer: trace.NewNoopTracer(), } - view := delta.NewView(state2.LedgerGetRegister(ledger, flow.StateCommitment(ledger.InitialState()))) + view := delta.NewDeltaView( + state2.NewLedgerStorageSnapshot( + ledger, + flow.StateCommitment(ledger.InitialState()))) blockView := view.NewChild() _, err = manager.ComputeBlock(context.Background(), computationResult.ExecutableBlock, blockView) @@ -223,7 +226,7 @@ func TestExecuteScript(t *testing.T) { ledger := testutil.RootBootstrappedLedger(vm, execCtx, fvm.WithExecutionMemoryLimit(math.MaxUint64)) - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) scriptView := view.NewChild() @@ -281,9 +284,10 @@ func TestExecuteScript_BalanceScriptFailsIfViewIsEmpty(t *testing.T) { me.On("SignFunc", mock.Anything, mock.Anything, mock.Anything). Return(nil, nil) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, fmt.Errorf("error getting register") - }) + view := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + return nil, fmt.Errorf("error getting register") + })) scriptView := view.NewChild() @@ -503,9 +507,7 @@ func (f *FakeBlockComputer) ExecuteBlock(context.Context, *entity.ExecutableBloc } func noopView() *delta.View { - return delta.NewView(func(_, _ string) (flow.RegisterValue, error) { - return nil, nil - }) + return delta.NewDeltaView(nil) } func TestExecuteScriptTimeout(t *testing.T) { @@ -697,7 +699,7 @@ func Test_EventEncodingFailsOnlyTxAndCarriesOn(t *testing.T) { tracer: trace.NewNoopTracer(), } - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) blockView := view.NewChild() eventEncoder.enabled = true diff --git a/engine/execution/computation/programs_test.go b/engine/execution/computation/programs_test.go index 87dac82866e..28d654bcd72 100644 --- a/engine/execution/computation/programs_test.go +++ b/engine/execution/computation/programs_test.go @@ -147,7 +147,7 @@ func TestPrograms_TestContractUpdates(t *testing.T) { derivedChainData: derivedChainData, } - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) blockView := view.NewChild() returnedComputationResult, err := engine.ComputeBlock(context.Background(), executableBlock, blockView) @@ -258,7 +258,7 @@ func TestPrograms_TestBlockForks(t *testing.T) { derivedChainData: derivedChainData, } - view := delta.NewView(ledger.Get) + view := delta.NewDeltaView(ledger) var ( res *execution.ComputationResult diff --git a/engine/execution/convert.go b/engine/execution/convert.go deleted file mode 100644 index 0ee02b8e955..00000000000 --- a/engine/execution/convert.go +++ /dev/null @@ -1,120 +0,0 @@ -package execution - -import ( - "fmt" - - "github.com/onflow/flow-go/model/convert" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" -) - -func GenerateExecutionResultAndChunkDataPacks( - metrics module.ExecutionMetrics, - prevResultId flow.Identifier, - startState flow.StateCommitment, - result *ComputationResult) ( - endState flow.StateCommitment, - chdps []*flow.ChunkDataPack, - executionResult *flow.ExecutionResult, - err error, -) { - - // no need to persist the state interactions, since they are used only by state - // syncing, which is currently disabled - block := result.ExecutableBlock.Block - blockID := block.ID() - - chunks := make([]*flow.Chunk, len(result.StateCommitments)) - chdps = make([]*flow.ChunkDataPack, len(result.StateCommitments)) - - // TODO: check current state root == startState - endState = startState - - for i := range result.StateCommitments { - // TODO: deltas should be applied to a particular state - - endState = result.StateCommitments[i] - var chunk *flow.Chunk - // account for system chunk being last - if i < len(result.StateCommitments)-1 { - // non-system chunks - collectionGuarantee := result.ExecutableBlock.Block.Payload.Guarantees[i] - completeCollection := result.ExecutableBlock.CompleteCollections[collectionGuarantee.ID()] - collection := completeCollection.Collection() - chunk = flow.NewChunk( - blockID, - i, - startState, - len(completeCollection.Transactions), - result.EventsHashes[i], - endState) - chdps[i] = flow.NewChunkDataPack( - chunk.ID(), - startState, - result.Proofs[i], - &collection) - metrics.ExecutionChunkDataPackGenerated(len(result.Proofs[i]), len(completeCollection.Transactions)) - - } else { - // system chunk - // note that system chunk does not have a collection. - // also, number of transactions is one for system chunk. - chunk = flow.NewChunk( - blockID, - i, - startState, - 1, - result.EventsHashes[i], - endState) - // system chunk has a nil collection. - chdps[i] = flow.NewChunkDataPack( - chunk.ID(), - startState, - result.Proofs[i], - nil) - metrics.ExecutionChunkDataPackGenerated(len(result.Proofs[i]), 1) - } - - // TODO use view.SpockSecret() as an input to spock generator - chunks[i] = chunk - startState = endState - } - - executionResult, err = GenerateExecutionResultForBlock(prevResultId, block, chunks, result.ServiceEvents, result.ExecutionDataID) - if err != nil { - return flow.DummyStateCommitment, nil, nil, fmt.Errorf("could not generate execution result: %w", err) - } - - return endState, chdps, executionResult, nil -} - -// GenerateExecutionResultForBlock creates new ExecutionResult for a block from -// the provided chunk results. -func GenerateExecutionResultForBlock( - previousErID flow.Identifier, - block *flow.Block, - chunks []*flow.Chunk, - serviceEvents []flow.Event, - executionDataID flow.Identifier, -) (*flow.ExecutionResult, error) { - - // convert Cadence service event representation to flow-go representation - convertedServiceEvents := make([]flow.ServiceEvent, 0, len(serviceEvents)) - for _, event := range serviceEvents { - converted, err := convert.ServiceEvent(block.Header.ChainID, event) - if err != nil { - return nil, fmt.Errorf("could not convert service event: %w", err) - } - convertedServiceEvents = append(convertedServiceEvents, *converted) - } - - er := &flow.ExecutionResult{ - PreviousResultID: previousErID, - BlockID: block.ID(), - Chunks: chunks, - ServiceEvents: convertedServiceEvents, - ExecutionDataID: executionDataID, - } - - return er, nil -} diff --git a/engine/execution/convert_test.go b/engine/execution/convert_test.go deleted file mode 100644 index fc804488708..00000000000 --- a/engine/execution/convert_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package execution_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/engine/execution" - executionUnittest "github.com/onflow/flow-go/engine/execution/state/unittest" - "github.com/onflow/flow-go/model/flow" - modulemock "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/utils/unittest" -) - -func Test_BuildChunkDataPack(t *testing.T) { - t.Run("number of transactions is included", func(t *testing.T) { - - // fixture provide one tx per collection, and number of collections equal to - // len of provided signersIDs - cr := executionUnittest.ComputationResultFixture([][]flow.Identifier{ - {flow.ZeroID}, - {flow.ZeroID}, - {flow.ZeroID}, - }) - - numberOfChunks := 4 - exemetrics := new(modulemock.ExecutionMetrics) - exemetrics.On("ExecutionChunkDataPackGenerated", - mock.Anything, - mock.Anything). - Return(nil). - Times(numberOfChunks) // 1 collection + system collection - - _, _, result, err := execution.GenerateExecutionResultAndChunkDataPacks(exemetrics, unittest.IdentifierFixture(), unittest.StateCommitmentFixture(), cr) - assert.NoError(t, err) - - require.Len(t, result.Chunks, numberOfChunks) // +1 for system chunk - - assert.Equal(t, uint64(1), result.Chunks[0].NumberOfTransactions) - assert.Equal(t, uint64(1), result.Chunks[1].NumberOfTransactions) - assert.Equal(t, uint64(1), result.Chunks[2].NumberOfTransactions) - - // system chunk is special case, but currently also 1 tx - assert.Equal(t, uint64(1), result.Chunks[3].NumberOfTransactions) - }) -} diff --git a/engine/execution/execution_test.go b/engine/execution/execution_test.go index ca91cfdf5c6..8585a54892c 100644 --- a/engine/execution/execution_test.go +++ b/engine/execution/execution_test.go @@ -101,7 +101,7 @@ func TestExecutionFlow(t *testing.T) { clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}) // signed by the only collector - block := unittest.BlockWithParentAndProposerFixture(genesis, conID.NodeID, 1) + block := unittest.BlockWithParentAndProposerFixture(t, genesis, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -126,7 +126,7 @@ func TestExecutionFlow(t *testing.T) { }, }) - child := unittest.BlockWithParentAndProposerFixture(block.Header, conID.NodeID, 1) + child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID) // the default signer indices is 2 bytes, but in this test cases // we need 1 byte child.Header.ParentVoterIndices = voterIndices @@ -262,7 +262,7 @@ func deployContractBlock(t *testing.T, conID *flow.Identity, colID *flow.Identit clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}) // make block - block := unittest.BlockWithParentAndProposerFixture(parent, conID.NodeID, 1) + block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -296,7 +296,7 @@ func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, ch clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}) // make block - block := unittest.BlockWithParentAndProposerFixture(parent, conID.NodeID, 1) + block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -329,7 +329,7 @@ func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}) col := &flow.Collection{Transactions: []*flow.TransactionBody{tx}} - block := unittest.BlockWithParentAndProposerFixture(parent, conID.NodeID, 1) + block := unittest.BlockWithParentAndProposerFixture(t, parent, conID.NodeID) voterIndices, err := signature.EncodeSignersToIndices( []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) @@ -543,16 +543,14 @@ func TestBroadcastToMultipleVerificationNodes(t *testing.T) { genesis, err := exeNode.State.AtHeight(0).Head() require.NoError(t, err) - block := unittest.BlockWithParentAndProposerFixture(genesis, conID.NodeID, 1) - voterIndices, err := signature.EncodeSignersToIndices( - []flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) + block := unittest.BlockWithParentAndProposerFixture(t, genesis, conID.NodeID) + voterIndices, err := signature.EncodeSignersToIndices([]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID}) require.NoError(t, err) block.Header.ParentVoterIndices = voterIndices - block.Header.View = 42 block.SetPayload(flow.Payload{}) proposal := unittest.ProposalFromBlock(&block) - child := unittest.BlockWithParentAndProposerFixture(block.Header, conID.NodeID, 1) + child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID) child.Header.ParentVoterIndices = voterIndices actualCalls := atomic.Uint64{} diff --git a/engine/execution/ingestion/engine.go b/engine/execution/ingestion/engine.go index 4f335ac13af..32a96588483 100644 --- a/engine/execution/ingestion/engine.go +++ b/engine/execution/ingestion/engine.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/rs/zerolog" @@ -146,10 +147,8 @@ func New( // successfully started. func (e *Engine) Ready() <-chan struct{} { if !e.stopControl.IsPaused() { - if e.uploader.Enabled() { - if err := e.uploader.RetryUploads(); err != nil { - e.log.Warn().Msg("failed to re-upload all ComputationResults") - } + if err := e.uploader.RetryUploads(); err != nil { + e.log.Warn().Msg("failed to re-upload all ComputationResults") } err := e.reloadUnexecutedBlocks() @@ -263,7 +262,7 @@ func (e *Engine) finalizedUnexecutedBlocks(finalized protocol.Snapshot) ([]flow. } func (e *Engine) pendingUnexecutedBlocks(finalized protocol.Snapshot) ([]flow.Identifier, error) { - pendings, err := finalized.ValidDescendants() + pendings, err := finalized.Descendants() if err != nil { return nil, fmt.Errorf("could not get pending blocks: %w", err) } @@ -428,7 +427,8 @@ func (e *Engine) reloadBlock( // BlockProcessable handles the new verified blocks (blocks that // have passed consensus validation) received from the consensus nodes -// Note: BlockProcessable might be called multiple times for the same block. +// NOTE: BlockProcessable might be called multiple times for the same block. +// NOTE: Ready calls reloadUnexecutedBlocks during initialization, which handles dropped protocol events. func (e *Engine) BlockProcessable(b *flow.Header) { // skip if stopControl tells to skip @@ -619,15 +619,20 @@ func (e *Engine) executeBlock(ctx context.Context, executableBlock *entity.Execu return } - if e.uploader.Enabled() { + wg := sync.WaitGroup{} + wg.Add(1) + defer wg.Wait() + + go func() { + defer wg.Done() err := e.uploader.Upload(ctx, computationResult) if err != nil { lg.Err(err).Msg("error while uploading block") // continue processing. uploads should not block execution } - } + }() - finalState, receipt, err := e.handleComputationResult(ctx, computationResult, *executableBlock.StartState) + finalState, receipt, err := e.handleComputationResult(ctx, computationResult) if errors.Is(err, storage.ErrDataMismatch) { lg.Fatal().Err(err).Msg("fatal: trying to store different results for the same block") } @@ -1105,9 +1110,10 @@ func (e *Engine) GetRegisterAtBlockID(ctx context.Context, owner, key []byte, bl blockView := e.execState.NewView(stateCommit) - data, err := blockView.Get(string(owner), string(key)) + id := flow.NewRegisterID(string(owner), string(key)) + data, err := blockView.Get(id) if err != nil { - return nil, fmt.Errorf("failed to get the register (owner : %s, key: %s): %w", hex.EncodeToString(owner), string(key), err) + return nil, fmt.Errorf("failed to get the register (%s): %w", id, err) } return data, nil @@ -1138,9 +1144,11 @@ func (e *Engine) GetAccount(ctx context.Context, addr flow.Address, blockID flow func (e *Engine) handleComputationResult( ctx context.Context, result *execution.ComputationResult, - startState flow.StateCommitment, -) (flow.StateCommitment, *flow.ExecutionReceipt, error) { - +) ( + flow.StateCommitment, + *flow.ExecutionReceipt, + error, +) { span, ctx := e.tracer.StartSpanFromContext(ctx, trace.EXEHandleComputationResult) defer span.End() @@ -1148,18 +1156,14 @@ func (e *Engine) handleComputationResult( Hex("block_id", logging.Entity(result.ExecutableBlock)). Msg("received computation result") - receipt, err := e.saveExecutionResults( - ctx, - result, - startState, - ) + receipt, err := e.saveExecutionResults(ctx, result) if err != nil { return flow.DummyStateCommitment, nil, fmt.Errorf("could not save execution results: %w", err) } finalState, err := receipt.ExecutionResult.FinalStateCommitment() if errors.Is(err, flow.ErrNoChunks) { - finalState = startState + finalState = *result.ExecutableBlock.StartState } else if err != nil { return flow.DummyStateCommitment, nil, fmt.Errorf("unexpected error accessing result's final state commitment: %w", err) } @@ -1170,14 +1174,13 @@ func (e *Engine) handleComputationResult( func (e *Engine) saveExecutionResults( ctx context.Context, result *execution.ComputationResult, - startState flow.StateCommitment, -) (*flow.ExecutionReceipt, error) { - +) ( + *flow.ExecutionReceipt, + error, +) { span, childCtx := e.tracer.StartSpanFromContext(ctx, trace.EXESaveExecutionResults) defer span.End() - originalState := startState - block := result.ExecutableBlock.Block previousErID, err := e.execState.GetExecutionResultID(ctx, block.Header.ParentID) if err != nil { @@ -1185,10 +1188,13 @@ func (e *Engine) saveExecutionResults( block.Header.ParentID, err) } - endState, chdps, executionResult, err := execution.GenerateExecutionResultAndChunkDataPacks(e.metrics, previousErID, startState, result) - if err != nil { - return nil, fmt.Errorf("cannot build chunk data pack: %w", err) - } + executionResult := flow.NewExecutionResult( + previousErID, + block.ID(), + result.Chunks, + result.ConvertedServiceEvents, + result.ExecutionDataID) + for _, event := range executionResult.ServiceEvents { e.log.Info(). Uint64("block_height", result.ExecutableBlock.Height()). @@ -1207,22 +1213,18 @@ func (e *Engine) saveExecutionResults( return nil, fmt.Errorf("could not generate execution receipt: %w", err) } - err = e.execState.SaveExecutionResults(childCtx, - block.Header, - endState, - chdps, - executionReceipt, - result.Events, - result.ServiceEvents, - result.TransactionResults) + err = e.execState.SaveExecutionResults( + childCtx, + result, + executionReceipt) if err != nil { return nil, fmt.Errorf("cannot persist execution state: %w", err) } e.log.Debug(). Hex("block_id", logging.Entity(result.ExecutableBlock)). - Hex("start_state", originalState[:]). - Hex("final_state", endState[:]). + Hex("start_state", result.ExecutableBlock.StartState[:]). + Hex("final_state", result.EndState[:]). Msg("saved computation results") return executionReceipt, nil diff --git a/engine/execution/ingestion/engine_test.go b/engine/execution/ingestion/engine_test.go index 38340698681..3ba73d7178b 100644 --- a/engine/execution/ingestion/engine_test.go +++ b/engine/execution/ingestion/engine_test.go @@ -23,10 +23,10 @@ import ( "github.com/onflow/flow-go/engine/execution/ingestion/uploader" uploadermock "github.com/onflow/flow-go/engine/execution/ingestion/uploader/mock" provider "github.com/onflow/flow-go/engine/execution/provider/mock" + "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/delta" - state "github.com/onflow/flow-go/engine/execution/state/mock" + stateMock "github.com/onflow/flow-go/engine/execution/state/mock" executionUnittest "github.com/onflow/flow-go/engine/execution/state/unittest" - "github.com/onflow/flow-go/engine/testutil/mocklocal" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -60,6 +60,52 @@ func init() { myIdentity.Role = flow.RoleExecution } +// ExecutionState is a mocked version of execution state that +// simulates some of its behavior for testing purpose +type mockExecutionState struct { + sync.Mutex + stateMock.ExecutionState + commits map[flow.Identifier]flow.StateCommitment +} + +func newMockExecutionState(seal *flow.Seal) *mockExecutionState { + commits := make(map[flow.Identifier]flow.StateCommitment) + commits[seal.BlockID] = seal.FinalState + return &mockExecutionState{ + commits: commits, + } +} + +func (es *mockExecutionState) StateCommitmentByBlockID( + ctx context.Context, + blockID flow.Identifier, +) ( + flow.StateCommitment, + error, +) { + es.Lock() + defer es.Unlock() + commit, ok := es.commits[blockID] + if !ok { + return flow.DummyStateCommitment, storageerr.ErrNotFound + } + + return commit, nil +} + +func (es *mockExecutionState) ExecuteBlock(t *testing.T, block *flow.Block) { + parentExecuted, err := state.IsBlockExecuted( + context.Background(), + es, + block.Header.ParentID) + require.NoError(t, err) + require.True(t, parentExecuted, "parent block not executed") + + es.Lock() + defer es.Unlock() + es.commits[block.ID()] = unittest.StateCommitmentFixture() +} + type testingContext struct { t *testing.T engine *Engine @@ -70,7 +116,7 @@ type testingContext struct { collectionConduit *mocknetwork.Conduit computationManager *computation.ComputationManager providerEngine *provider.ProviderEngine - executionState *state.ExecutionState + executionState *stateMock.ExecutionState snapshot *protocol.Snapshot identity *flow.Identity broadcastedReceipts map[flow.Identifier]*flow.ExecutionReceipt @@ -114,7 +160,7 @@ func runWithEngine(t *testing.T, f func(testingContext)) { computationManager := new(computation.ComputationManager) providerEngine := new(provider.ProviderEngine) protocolState := new(protocol.State) - executionState := new(state.ExecutionState) + executionState := new(stateMock.ExecutionState) snapshot := new(protocol.Snapshot) var engine *Engine @@ -254,30 +300,19 @@ func (ctx *testingContext) assertSuccessfulBlockComputation( mocked := ctx.executionState. On("SaveExecutionResults", mock.Anything, - executableBlock.Block.Header, - newStateCommitment, - mock.MatchedBy(func(fs []*flow.ChunkDataPack) bool { - for _, f := range fs { - if f.StartState != *executableBlock.StartState { - return false - } - } - return true - }), + computationResult, mock.MatchedBy(func(executionReceipt *flow.ExecutionReceipt) bool { return executionReceipt.ExecutionResult.BlockID == executableBlock.Block.ID() && executionReceipt.ExecutionResult.PreviousResultID == previousExecutionResultID }), - mock.Anything, - mock.Anything, - mock.Anything, ). Return(nil) mocked.RunFn = func(args mock.Arguments) { - blockID := args[1].(*flow.Header).ID() - commit := args[2].(flow.StateCommitment) + result := args[1].(*execution.ComputationResult) + blockID := result.ExecutableBlock.Block.Header.ID() + commit := result.EndState ctx.mu.Lock() commits[blockID] = commit @@ -575,7 +610,7 @@ func Test_OnlyHeadOfTheQueueIsExecuted(t *testing.T) { blockASnapshot.On("Head").Return(&blockA, nil) blockCSnapshot.On("Head").Return(blockC.Block.Header, nil) - blockCSnapshot.On("ValidDescendants").Return(nil, nil) + blockCSnapshot.On("Descendants").Return(nil, nil) ctx.state.On("AtHeight", blockC.Height()).Return(blockCSnapshot) @@ -1141,13 +1176,12 @@ func TestStopAtHeightRaceFinalization(t *testing.T) { func TestExecutionGenerationResultsAreChained(t *testing.T) { - execState := new(state.ExecutionState) + execState := new(stateMock.ExecutionState) ctrl := gomock.NewController(t) me := module.NewMockLocal(ctrl) executableBlock := unittest.ExecutableBlockFixture([][]flow.Identifier{{collection1Identity.NodeID}, {collection1Identity.NodeID}}) - startState := unittest.StateCommitmentFixture() previousExecutionResultID := unittest.IdentifierFixture() // mock execution state conversion and signing of @@ -1155,12 +1189,17 @@ func TestExecutionGenerationResultsAreChained(t *testing.T) { me.EXPECT().NodeID() me.EXPECT().Sign(gomock.Any(), gomock.Any()) + cr := executionUnittest.ComputationResultFixture(nil) + cr.ExecutableBlock = executableBlock + startState := unittest.StateCommitmentFixture() + cr.ExecutableBlock.StartState = &startState + execState. On("GetExecutionResultID", mock.Anything, executableBlock.Block.Header.ParentID). Return(previousExecutionResultID, nil) execState. - On("SaveExecutionResults", mock.Anything, executableBlock.Block.Header, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + On("SaveExecutionResults", mock.Anything, cr, mock.Anything). Return(nil) e := Engine{ @@ -1170,10 +1209,7 @@ func TestExecutionGenerationResultsAreChained(t *testing.T) { me: me, } - cr := executionUnittest.ComputationResultFixture(nil) - cr.ExecutableBlock = executableBlock - - er, err := e.saveExecutionResults(context.Background(), cr, startState) + er, err := e.saveExecutionResults(context.Background(), cr) assert.NoError(t, err) assert.Equal(t, previousExecutionResultID, er.ExecutionResult.PreviousResultID) @@ -1371,7 +1407,7 @@ func TestUnauthorizedNodeDoesNotBroadcastReceipts(t *testing.T) { // require.True(t, shouldTriggerStateSync(20, 29, 10)) // } -func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mocks.ExecutionState) *Engine { +func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mockExecutionState) *Engine { log := unittest.Logger() metrics := metrics.NewNoopCollector() tracer, err := trace.NewTracer(log, "test", "test", trace.SensitivityCaptureAll) @@ -1458,7 +1494,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Bootstrap(genesis, result, seal)) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) finalized, pending, err := engine.unexecutedBlocks() @@ -1483,7 +1519,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Extend(blockC)) require.NoError(t, ps.Extend(blockD)) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) finalized, pending, err := engine.unexecutedBlocks() @@ -1508,7 +1544,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Extend(blockC)) require.NoError(t, ps.Extend(blockD)) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) es.ExecuteBlock(t, blockA) @@ -1538,7 +1574,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockC.ID())) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) es.ExecuteBlock(t, blockA) @@ -1569,7 +1605,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockC.ID())) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) es.ExecuteBlock(t, blockA) @@ -1599,7 +1635,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Extend(blockD)) require.NoError(t, ps.Finalize(blockA.ID())) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) es.ExecuteBlock(t, blockA) @@ -1654,7 +1690,7 @@ func TestLoadingUnexecutedBlocks(t *testing.T) { require.NoError(t, ps.Finalize(blockC.ID())) - es := mocks.NewExecutionState(seal) + es := newMockExecutionState(seal) engine := newIngestionEngine(t, ps, es) diff --git a/engine/execution/ingestion/uploader/manager.go b/engine/execution/ingestion/uploader/manager.go index 108a4a295a0..747fcd18a17 100644 --- a/engine/execution/ingestion/uploader/manager.go +++ b/engine/execution/ingestion/uploader/manager.go @@ -11,7 +11,8 @@ import ( "github.com/onflow/flow-go/module/trace" ) -// Manager encapsulates the logic for uploading computation results to cloud storage. It +// Manager encapsulates the logic for uploading computation results to cloud +// storage. type Manager struct { enabled bool tracer module.Tracer @@ -53,10 +54,17 @@ func (m *Manager) Enabled() bool { // Upload uploads the given computation result with all uploaders // Any errors returned by the uploaders may be considered benign -func (m *Manager) Upload(ctx context.Context, result *execution.ComputationResult) error { +func (m *Manager) Upload( + ctx context.Context, + result *execution.ComputationResult, +) error { m.mu.RLock() defer m.mu.RUnlock() + if !m.enabled { + return nil + } + var group errgroup.Group for _, uploader := range m.uploaders { @@ -79,6 +87,10 @@ func (m *Manager) RetryUploads() (err error) { m.mu.RLock() defer m.mu.RUnlock() + if !m.enabled { + return nil + } + for _, u := range m.uploaders { switch retryableUploaderWraper := u.(type) { case RetryableUploaderWrapper: diff --git a/engine/execution/messages.go b/engine/execution/messages.go index f562bd78f85..f225dbefc4d 100644 --- a/engine/execution/messages.go +++ b/engine/execution/messages.go @@ -14,12 +14,7 @@ import ( // TODO If the executor will be a separate process/machine we would need to rework // sending view as local data, but that would be much greater refactor of storage anyway -type ComputationOrder struct { - Block *entity.ExecutableBlock - View *delta.View - StartState flow.StateCommitment -} - +// TODO(patrick): rm unaccessed fields type ComputationResult struct { ExecutableBlock *entity.ExecutableBlock StateSnapshots []*delta.SpockSnapshot @@ -28,27 +23,39 @@ type ComputationResult struct { Events []flow.EventsList EventsHashes []flow.Identifier ServiceEvents flow.EventsList + ConvertedServiceEvents flow.ServiceEventList TransactionResults []flow.TransactionResult TransactionResultIndex []int ComputationIntensities meter.MeteredComputationIntensities TrieUpdates []*ledger.TrieUpdate ExecutionDataID flow.Identifier SpockSignatures []crypto.Signature + Chunks []*flow.Chunk + ChunkDataPacks []*flow.ChunkDataPack + EndState flow.StateCommitment } -func NewEmptyComputationResult(block *entity.ExecutableBlock) *ComputationResult { +func NewEmptyComputationResult( + block *entity.ExecutableBlock, +) *ComputationResult { numCollections := len(block.CompleteCollections) + 1 return &ComputationResult{ ExecutableBlock: block, + StateSnapshots: make([]*delta.SpockSnapshot, 0, numCollections), + StateCommitments: make([]flow.StateCommitment, 0, numCollections), + Proofs: make([][]byte, 0, numCollections), Events: make([]flow.EventsList, numCollections), + EventsHashes: make([]flow.Identifier, 0, numCollections), ServiceEvents: make(flow.EventsList, 0), + ConvertedServiceEvents: make(flow.ServiceEventList, 0), TransactionResults: make([]flow.TransactionResult, 0), TransactionResultIndex: make([]int, 0), - StateCommitments: make([]flow.StateCommitment, 0, numCollections), - Proofs: make([][]byte, 0, numCollections), - TrieUpdates: make([]*ledger.TrieUpdate, 0, numCollections), - EventsHashes: make([]flow.Identifier, 0, numCollections), ComputationIntensities: make(meter.MeteredComputationIntensities), + TrieUpdates: make([]*ledger.TrieUpdate, 0, numCollections), + SpockSignatures: make([]crypto.Signature, 0, numCollections), + Chunks: make([]*flow.Chunk, 0, numCollections), + ChunkDataPacks: make([]*flow.ChunkDataPack, 0, numCollections), + EndState: *block.StartState, } } @@ -60,6 +67,9 @@ func (cr *ComputationResult) AddTransactionResult( cr.Events[collectionIndex], txn.Events...) cr.ServiceEvents = append(cr.ServiceEvents, txn.ServiceEvents...) + cr.ConvertedServiceEvents = append( + cr.ConvertedServiceEvents, + txn.ConvertedServiceEvents...) txnResult := flow.TransactionResult{ TransactionID: txn.ID, diff --git a/engine/execution/rpc/engine.go b/engine/execution/rpc/engine.go index c28b18f13a7..d4ec6aa3198 100644 --- a/engine/execution/rpc/engine.go +++ b/engine/execution/rpc/engine.go @@ -24,13 +24,12 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/grpcutils" ) // Config defines the configurable options for the gRPC server. type Config struct { ListenAddr string - MaxMsgSize int // In bytes + MaxMsgSize uint // In bytes RpcMetricsEnabled bool // enable GRPC metrics reporting } @@ -60,13 +59,9 @@ func New( apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the gRPC API e.g. Ping->50, ExecuteScriptAtBlockID->10 ) *Engine { log = log.With().Str("engine", "rpc").Logger() - if config.MaxMsgSize == 0 { - config.MaxMsgSize = grpcutils.DefaultMaxMsgSize - } - serverOptions := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(config.MaxMsgSize), - grpc.MaxSendMsgSize(config.MaxMsgSize), + grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxMsgSize)), } var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors @@ -272,13 +267,13 @@ func (h *handler) GetTransactionResult( reqBlockID := req.GetBlockId() blockID, err := convert.BlockID(reqBlockID) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid blockID: %v", err) } reqTxID := req.GetTransactionId() txID, err := convert.TransactionID(reqTxID) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid transactionID: %v", err) } var statusCode uint32 = 0 @@ -334,7 +329,7 @@ func (h *handler) GetTransactionResultByIndex( reqBlockID := req.GetBlockId() blockID, err := convert.BlockID(reqBlockID) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid blockID: %v", err) } index := req.GetIndex() @@ -392,7 +387,7 @@ func (h *handler) GetTransactionResultsByBlockID( reqBlockID := req.GetBlockId() blockID, err := convert.BlockID(reqBlockID) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid blockID: %v", err) } // Get all tx results @@ -488,15 +483,18 @@ func (h *handler) GetAccountAtBlockID( blockID := req.GetBlockId() blockFlowID, err := convert.BlockID(blockID) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid blockID: %v", err) } flowAddress, err := convert.Address(req.GetAddress(), h.chain.Chain()) if err != nil { - return nil, err + return nil, status.Errorf(codes.InvalidArgument, "invalid address: %v", err) } value, err := h.engine.GetAccount(ctx, flowAddress, blockFlowID) + if errors.Is(err, storage.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "account with address %s not found", flowAddress) + } if err != nil { return nil, status.Errorf(codes.Internal, "failed to get account: %v", err) } @@ -559,6 +557,7 @@ func (h *handler) GetBlockHeaderByID( func (h *handler) blockHeaderResponse(header *flow.Header) (*execution.BlockHeaderResponse, error) { signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(header) if err != nil { + // the block was retrieved from local storage - so no errors are expected return nil, fmt.Errorf("failed to decode signer indices to Identifiers for block %v: %w", header.ID(), err) } diff --git a/engine/execution/state/bootstrap/bootstrap.go b/engine/execution/state/bootstrap/bootstrap.go index 36fd3aeea13..ba0ee3e39ee 100644 --- a/engine/execution/state/bootstrap/bootstrap.go +++ b/engine/execution/state/bootstrap/bootstrap.go @@ -36,7 +36,10 @@ func (b *Bootstrapper) BootstrapLedger( chain flow.Chain, opts ...fvm.BootstrapProcedureOption, ) (flow.StateCommitment, error) { - view := delta.NewView(state.LedgerGetRegister(ledger, flow.StateCommitment(ledger.InitialState()))) + view := delta.NewDeltaView( + state.NewLedgerStorageSnapshot( + ledger, + flow.StateCommitment(ledger.InitialState()))) vm := fvm.NewVirtualMachine() diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 097fc88bdba..dd05220189a 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("8265f0fc3b74daa0869c1cc4fbc9ec0af2594d07657cf4ca39440537e2a12b13") + expectedStateCommitmentBytes, _ := hex.DecodeString("263034d3d5606ed242c766cd22b14ad53d11cdab1eb4b65a6c904c2a25a71fe7") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/engine/execution/state/delta/delta.go b/engine/execution/state/delta/delta.go index 49f62a3a3c5..524555c4e54 100644 --- a/engine/execution/state/delta/delta.go +++ b/engine/execution/state/delta/delta.go @@ -1,9 +1,6 @@ package delta import ( - "encoding/json" - "fmt" - "golang.org/x/exp/slices" "github.com/onflow/flow-go/model/flow" @@ -25,19 +22,28 @@ func NewDelta() Delta { // // This function will return nil if the given key has been deleted in this delta. // Second return parameters indicated if the value has been set/deleted in this delta -func (d Delta) Get(owner, key string) (flow.RegisterValue, bool) { - value, set := d.Data[flow.NewRegisterID(owner, key)] +func (d Delta) Get(id flow.RegisterID) (flow.RegisterValue, bool) { + value, set := d.Data[id] return value, set } // Set records an update in this delta. -func (d Delta) Set(owner, key string, value flow.RegisterValue) { - k := flow.NewRegisterID(owner, key) - d.Data[k] = value +func (d Delta) Set(id flow.RegisterID, value flow.RegisterValue) { + d.Data[id] = value +} + +// UpdatedRegisterIDs returns all register ids that were updated by this delta. +// The returned ids are unsorted. +func (d Delta) UpdatedRegisterIDs() []flow.RegisterID { + ids := make([]flow.RegisterID, 0, len(d.Data)) + for key := range d.Data { + ids = append(ids, key) + } + return ids } // UpdatedRegisters returns all registers that were updated by this delta. -// ids are returned sorted, in ascending order +// The returned entries are sorted by ids in ascending order. func (d Delta) UpdatedRegisters() flow.RegisterEntries { entries := make(flow.RegisterEntries, 0, len(d.Data)) for key, value := range d.Data { @@ -85,30 +91,3 @@ func (d Delta) RegisterIDs() []flow.RegisterID { } return ids } - -func (d Delta) MarshalJSON() ([]byte, error) { - m := make(flow.RegisterEntries, len(d.Data)) - for key, value := range d.Data { - m = append(m, flow.RegisterEntry{Key: key, Value: value}) - } - return json.Marshal(m) -} - -func (d *Delta) UnmarshalJSON(data []byte) error { - - var m flow.RegisterEntries - - err := json.Unmarshal(data, &m) - if err != nil { - return fmt.Errorf("cannot umarshal Delta: %w", err) - } - dd := make(map[flow.RegisterID]flow.RegisterValue, len(m)) - - for _, value := range m { - dd[value.Key] = value.Value - } - - d.Data = dd - - return nil -} diff --git a/engine/execution/state/delta/delta_test.go b/engine/execution/state/delta/delta_test.go index 9766f159046..706f57cd79e 100644 --- a/engine/execution/state/delta/delta_test.go +++ b/engine/execution/state/delta/delta_test.go @@ -11,12 +11,12 @@ import ( ) func TestDelta_Get(t *testing.T) { - registerID1 := "fruit" + registerID1 := flow.NewRegisterID("fruit", "") t.Run("ValueNotSet", func(t *testing.T) { d := delta.NewDelta() - b, exists := d.Get(registerID1, "") + b, exists := d.Get(registerID1) assert.Nil(t, b) assert.False(t, exists) }) @@ -24,50 +24,50 @@ func TestDelta_Get(t *testing.T) { t.Run("ValueSet", func(t *testing.T) { d := delta.NewDelta() - d.Set(registerID1, "", []byte("apple")) + d.Set(registerID1, []byte("apple")) - b, exists := d.Get(registerID1, "") + b, exists := d.Get(registerID1) assert.Equal(t, flow.RegisterValue("apple"), b) assert.True(t, exists) }) } func TestDelta_Set(t *testing.T) { - registerID1 := "fruit" + registerID1 := flow.NewRegisterID("fruit", "") d := delta.NewDelta() - d.Set(registerID1, "", []byte("apple")) + d.Set(registerID1, []byte("apple")) - b1, exists := d.Get(registerID1, "") + b1, exists := d.Get(registerID1) assert.Equal(t, []byte("apple"), b1) assert.True(t, exists) - d.Set(registerID1, "", []byte("orange")) + d.Set(registerID1, []byte("orange")) - b2, exists := d.Get(registerID1, "") + b2, exists := d.Get(registerID1) assert.Equal(t, []byte("orange"), b2) assert.True(t, exists) } func TestDelta_MergeWith(t *testing.T) { - registerID1 := "fruit" + registerID1 := flow.NewRegisterID("fruit", "") - registerID2 := "vegetable" + registerID2 := flow.NewRegisterID("vegetable", "") t.Run("NoCollisions", func(t *testing.T) { d1 := delta.NewDelta() d2 := delta.NewDelta() - d1.Set(registerID1, "", []byte("apple")) - d2.Set(registerID2, "", []byte("carrot")) + d1.Set(registerID1, []byte("apple")) + d2.Set(registerID2, []byte("carrot")) d1.MergeWith(d2) - b1, _ := d1.Get(registerID1, "") + b1, _ := d1.Get(registerID1) assert.Equal(t, flow.RegisterValue("apple"), b1) - b2, _ := d1.Get(registerID2, "") + b2, _ := d1.Get(registerID2) assert.Equal(t, flow.RegisterValue("carrot"), b2) }) @@ -75,12 +75,12 @@ func TestDelta_MergeWith(t *testing.T) { d1 := delta.NewDelta() d2 := delta.NewDelta() - d1.Set(registerID1, "", flow.RegisterValue("apple")) - d2.Set(registerID1, "", flow.RegisterValue("orange")) + d1.Set(registerID1, flow.RegisterValue("apple")) + d2.Set(registerID1, flow.RegisterValue("orange")) d1.MergeWith(d2) - b, _ := d1.Get(registerID1, "") + b, _ := d1.Get(registerID1) assert.Equal(t, flow.RegisterValue("orange"), b) }) @@ -88,14 +88,14 @@ func TestDelta_MergeWith(t *testing.T) { d1 := delta.NewDelta() d2 := delta.NewDelta() - d1.Set(registerID1, "", flow.RegisterValue("apple")) - d1.Set(registerID1, "", nil) + d1.Set(registerID1, flow.RegisterValue("apple")) + d1.Set(registerID1, nil) - d2.Set(registerID1, "", flow.RegisterValue("orange")) + d2.Set(registerID1, flow.RegisterValue("orange")) d1.MergeWith(d2) - b, _ := d1.Get(registerID1, "") + b, _ := d1.Get(registerID1) assert.Equal(t, flow.RegisterValue("orange"), b) }) @@ -103,13 +103,13 @@ func TestDelta_MergeWith(t *testing.T) { d1 := delta.NewDelta() d2 := delta.NewDelta() - d1.Set(registerID1, "", flow.RegisterValue("apple")) + d1.Set(registerID1, flow.RegisterValue("apple")) - d2.Set(registerID1, "", nil) + d2.Set(registerID1, nil) d1.MergeWith(d2) - b, exists := d1.Get(registerID1, "") + b, exists := d1.Get(registerID1) assert.Nil(t, b) assert.True(t, exists) }) @@ -136,11 +136,11 @@ func TestDelta_UpdatedRegistersAreSorted(t *testing.T) { sort.Sort(data) // set in random order - d.Set(data[2].Key.Owner, data[2].Key.Key, data[2].Value) - d.Set(data[1].Key.Owner, data[1].Key.Key, data[1].Value) - d.Set(data[3].Key.Owner, data[3].Key.Key, data[3].Value) - d.Set(data[0].Key.Owner, data[0].Key.Key, data[0].Value) - d.Set(data[4].Key.Owner, data[4].Key.Key, data[4].Value) + d.Set(data[2].Key, data[2].Value) + d.Set(data[1].Key, data[1].Value) + d.Set(data[3].Key, data[3].Value) + d.Set(data[0].Key, data[0].Value) + d.Set(data[4].Key, data[4].Value) ret := d.UpdatedRegisters() diff --git a/engine/execution/state/delta/storage_snapshot.go b/engine/execution/state/delta/storage_snapshot.go new file mode 100644 index 00000000000..494be285263 --- /dev/null +++ b/engine/execution/state/delta/storage_snapshot.go @@ -0,0 +1,51 @@ +package delta + +import ( + "github.com/onflow/flow-go/model/flow" +) + +type StorageSnapshot interface { + // Get returns the register id's value, or an empty RegisterValue if the id + // is not found. + Get(id flow.RegisterID) (flow.RegisterValue, error) +} + +type EmptyStorageSnapshot struct{} + +func (EmptyStorageSnapshot) Get( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + return nil, nil +} + +type ReadFuncStorageSnapshot struct { + ReadFunc func(flow.RegisterID) (flow.RegisterValue, error) +} + +func NewReadFuncStorageSnapshot( + readFunc func(flow.RegisterID) (flow.RegisterValue, error), +) StorageSnapshot { + return &ReadFuncStorageSnapshot{ + ReadFunc: readFunc, + } +} + +func (storage ReadFuncStorageSnapshot) Get( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + return storage.ReadFunc(id) +} + +type Peeker interface { + Peek(id flow.RegisterID) (flow.RegisterValue, error) +} + +func NewPeekerStorageSnapshot(peeker Peeker) StorageSnapshot { + return NewReadFuncStorageSnapshot(peeker.Peek) +} diff --git a/engine/execution/state/delta/view.go b/engine/execution/state/delta/view.go index 5b7cb330a90..d2cfd8767b4 100644 --- a/engine/execution/state/delta/view.go +++ b/engine/execution/state/delta/view.go @@ -9,9 +9,6 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// GetRegisterFunc is a function that returns the value for a register. -type GetRegisterFunc func(owner, key string) (flow.RegisterValue, error) - // A View is a read-only view into a ledger stored in an underlying data source. // // A ledger view records writes to a delta that can be used to update the @@ -26,7 +23,8 @@ type View struct { spockSecret []byte spockSecretLock *sync.Mutex // using pointer instead, because using value would cause mock.Called to trigger race detector spockSecretHasher hash.Hasher - readFunc GetRegisterFunc + + storage StorageSnapshot } type Snapshot struct { @@ -46,17 +44,28 @@ type SpockSnapshot struct { SpockSecret []byte } -func AlwaysEmptyGetRegisterFunc(owner, key string) (flow.RegisterValue, error) { - return nil, nil +// TODO(patrick): rm after updating emulator. +func NewView( + readFunc func(owner string, key string) (flow.RegisterValue, error), +) *View { + return NewDeltaView( + ReadFuncStorageSnapshot{ + ReadFunc: func(id flow.RegisterID) (flow.RegisterValue, error) { + return readFunc(id.Owner, id.Key) + }, + }) } -// NewView instantiates a new ledger view with the provided read function. -func NewView(readFunc GetRegisterFunc) *View { +// NewDeltaView instantiates a new ledger view with the provided read function. +func NewDeltaView(storage StorageSnapshot) *View { + if storage == nil { + storage = EmptyStorageSnapshot{} + } return &View{ delta: NewDelta(), spockSecretLock: &sync.Mutex{}, regTouchSet: make(map[flow.RegisterID]struct{}), - readFunc: readFunc, + storage: storage, spockSecretHasher: hash.NewSHA3_256(), } } @@ -93,8 +102,9 @@ func (v *View) Interactions() *SpockSnapshot { } } -// AllRegisters returns all the register IDs either in read or delta -func (r *Snapshot) AllRegisters() []flow.RegisterID { +// AllRegisterIDs returns all the register IDs either in read or delta. +// The returned ids are unsorted. +func (r *Snapshot) AllRegisterIDs() []flow.RegisterID { set := make(map[flow.RegisterID]struct{}, len(r.Reads)+len(r.Delta.Data)) for reg := range r.Reads { set[reg] = struct{}{} @@ -111,18 +121,23 @@ func (r *Snapshot) AllRegisters() []flow.RegisterID { // NewChild generates a new child view, with the current view as the base, sharing the Get function func (v *View) NewChild() state.View { - return NewView(v.Peek) + return NewDeltaView(NewPeekerStorageSnapshot(v)) } func (v *View) DropDelta() { v.delta = NewDelta() } -func (v *View) AllRegisters() []flow.RegisterID { - return v.Interactions().AllRegisters() +func (v *View) AllRegisterIDs() []flow.RegisterID { + return v.Interactions().AllRegisterIDs() } -// RegisterUpdates returns a list of register updates +// UpdatedRegisterIDs returns a list of updated registers' ids. +func (v *View) UpdatedRegisterIDs() []flow.RegisterID { + return v.Delta().UpdatedRegisterIDs() +} + +// UpdatedRegisters returns a list of updated registers. func (v *View) UpdatedRegisters() flow.RegisterEntries { return v.Delta().UpdatedRegisters() } @@ -131,13 +146,12 @@ func (v *View) UpdatedRegisters() flow.RegisterEntries { // // This function will return an error if it fails to read from the underlying // data source for this view. -func (v *View) Get(owner, key string) (flow.RegisterValue, error) { +func (v *View) Get(registerID flow.RegisterID) (flow.RegisterValue, error) { var err error - registerID := flow.NewRegisterID(owner, key) - value, exists := v.delta.Get(owner, key) + value, exists := v.delta.Get(registerID) if !exists { - value, err = v.readFunc(owner, key) + value, err = v.storage.Get(registerID) if err != nil { return nil, fmt.Errorf("get register failed: %w", err) } @@ -156,18 +170,17 @@ func (v *View) Get(owner, key string) (flow.RegisterValue, error) { } // Peek reads the value without registering the read, as when used as parent read function -func (v *View) Peek(owner, key string) (flow.RegisterValue, error) { - value, exists := v.delta.Get(owner, key) +func (v *View) Peek(id flow.RegisterID) (flow.RegisterValue, error) { + value, exists := v.delta.Get(id) if exists { return value, nil } - return v.readFunc(owner, key) + return v.storage.Get(id) } // Set sets a register value in this view. -func (v *View) Set(owner, key string, value flow.RegisterValue) error { - registerID := flow.NewRegisterID(owner, key) +func (v *View) Set(registerID flow.RegisterID, value flow.RegisterValue) error { // every time we write something to delta (order preserving) we update // the spock secret with both the register ID and value. @@ -184,28 +197,10 @@ func (v *View) Set(owner, key string, value flow.RegisterValue) error { // capture register touch v.regTouchSet[registerID] = struct{}{} // add key value to delta - v.delta.Set(owner, key, value) + v.delta.Set(registerID, value) return nil } -// Touch explicitly adds a register to the touched registers set. -func (v *View) Touch(owner, key string) error { - - k := flow.NewRegisterID(owner, key) - - // capture register touch - v.regTouchSet[k] = struct{}{} - // increase reads - v.readsCount++ - - return nil -} - -// Delete removes a register in this view. -func (v *View) Delete(owner, key string) error { - return v.Set(owner, key, nil) -} - // Delta returns a record of the registers that were mutated in this view. func (v *View) Delta() Delta { return v.delta @@ -269,9 +264,3 @@ func (v *View) SpockSecret() []byte { v.spockSecretLock.Unlock() return v.spockSecret } - -// Detach detaches view from parent, by setting readFunc to -// default, empty one -func (v *View) Detach() { - v.readFunc = AlwaysEmptyGetRegisterFunc -} diff --git a/engine/execution/state/delta/view_test.go b/engine/execution/state/delta/view_test.go index 513598998d2..3a97411e01e 100644 --- a/engine/execution/state/delta/view_test.go +++ b/engine/execution/state/delta/view_test.go @@ -11,155 +11,132 @@ import ( "github.com/onflow/flow-go/model/flow" ) +type testStorage map[flow.RegisterID]string + +func (storage testStorage) Get(id flow.RegisterID) (flow.RegisterValue, error) { + return flow.RegisterValue(storage[id]), nil +} + func TestViewGet(t *testing.T) { - registerID := "fruit" + registerID := flow.NewRegisterID("fruit", "") t.Run("ValueNotSet", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - b, err := v.Get(registerID, "") + b, err := v.Get(registerID) assert.NoError(t, err) assert.Nil(t, b) }) t.Run("ValueNotInCache", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == registerID { - return flow.RegisterValue("orange"), nil - } - - return nil, nil - }) - b, err := v.Get(registerID, "") + v := delta.NewDeltaView( + testStorage{ + registerID: "orange", + }) + b, err := v.Get(registerID) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("orange"), b) }) t.Run("ValueInCache", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == registerID { - return flow.RegisterValue("orange"), nil - } - - return nil, nil - }) - - err := v.Set(registerID, "", flow.RegisterValue("apple")) + v := delta.NewDeltaView( + testStorage{ + registerID: "orange", + }) + err := v.Set(registerID, flow.RegisterValue("apple")) assert.NoError(t, err) - b, err := v.Get(registerID, "") + b, err := v.Get(registerID) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b) }) } func TestViewSet(t *testing.T) { - registerID := "fruit" + registerID := flow.NewRegisterID("fruit", "") - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - err := v.Set(registerID, "", flow.RegisterValue("apple")) + err := v.Set(registerID, flow.RegisterValue("apple")) assert.NoError(t, err) - b1, err := v.Get(registerID, "") + b1, err := v.Get(registerID) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) - err = v.Set(registerID, "", flow.RegisterValue("orange")) + err = v.Set(registerID, flow.RegisterValue("orange")) assert.NoError(t, err) - b2, err := v.Get(registerID, "") + b2, err := v.Get(registerID) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("orange"), b2) - t.Run("AfterDelete", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + t.Run("Overwrite register", func(t *testing.T) { + v := delta.NewDeltaView(nil) - err := v.Set(registerID, "", flow.RegisterValue("apple")) - assert.NoError(t, err) - err = v.Delete(registerID, "") + err := v.Set(registerID, flow.RegisterValue("apple")) assert.NoError(t, err) - err = v.Set(registerID, "", flow.RegisterValue("orange")) + err = v.Set(registerID, flow.RegisterValue("orange")) assert.NoError(t, err) - b, err := v.Get(registerID, "") + b, err := v.Get(registerID) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("orange"), b) }) t.Run("SpockSecret", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) t.Run("reflects in the snapshot", func(t *testing.T) { assert.Equal(t, v.SpockSecret(), v.Interactions().SpockSecret) }) - v = delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v = delta.NewDeltaView(nil) - registerID1 := "reg1" - registerID2 := "reg2" - registerID3 := "reg3" + registerID1 := flow.NewRegisterID("reg1", "") + registerID2 := flow.NewRegisterID("reg2", "") + registerID3 := flow.NewRegisterID("reg3", "") // prepare the registerID bytes - register := flow.NewRegisterID("", "") - register.Owner = registerID1 - registerID1Bytes := register.Bytes() - register.Owner = registerID2 - registerID2Bytes := register.Bytes() - register.Owner = registerID3 - registerID3Bytes := register.Bytes() + registerID1Bytes := registerID1.Bytes() + registerID2Bytes := registerID2.Bytes() + registerID3Bytes := registerID3.Bytes() // this part checks that spocks ordering be based // on update orders and not registerIDs expSpock := hash.NewSHA3_256() - err = v.Set(registerID2, "", flow.RegisterValue("1")) + err = v.Set(registerID2, flow.RegisterValue("1")) require.NoError(t, err) hashIt(t, expSpock, registerID2Bytes) hashIt(t, expSpock, []byte("1")) - err = v.Set(registerID3, "", flow.RegisterValue("2")) + err = v.Set(registerID3, flow.RegisterValue("2")) require.NoError(t, err) hashIt(t, expSpock, registerID3Bytes) hashIt(t, expSpock, []byte("2")) - err = v.Set(registerID1, "", flow.RegisterValue("3")) + err = v.Set(registerID1, flow.RegisterValue("3")) require.NoError(t, err) hashIt(t, expSpock, registerID1Bytes) hashIt(t, expSpock, []byte("3")) - _, err := v.Get(registerID1, "") - require.NoError(t, err) - hashIt(t, expSpock, registerID1Bytes) - - // this part uses the delete functionality - // to check that only the register ID is written to the spock secret - err = v.Delete(registerID1, "") + _, err := v.Get(registerID1) require.NoError(t, err) hashIt(t, expSpock, registerID1Bytes) // this part checks that it always update the // intermediate values and not just the final values - err = v.Set(registerID1, "", flow.RegisterValue("4")) + err = v.Set(registerID1, flow.RegisterValue("4")) require.NoError(t, err) hashIt(t, expSpock, registerID1Bytes) hashIt(t, expSpock, []byte("4")) - err = v.Set(registerID1, "", flow.RegisterValue("5")) + err = v.Set(registerID1, flow.RegisterValue("5")) require.NoError(t, err) hashIt(t, expSpock, registerID1Bytes) hashIt(t, expSpock, []byte("5")) - err = v.Set(registerID3, "", flow.RegisterValue("6")) + err = v.Set(registerID3, flow.RegisterValue("6")) require.NoError(t, err) hashIt(t, expSpock, registerID3Bytes) hashIt(t, expSpock, []byte("6")) @@ -173,199 +150,117 @@ func TestViewSet(t *testing.T) { }) } -func TestView_Delete(t *testing.T) { - registerID := "fruit" - - t.Run("ValueNotSet", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) - - b1, err := v.Get(registerID, "") - assert.NoError(t, err) - assert.Nil(t, b1) - - err = v.Delete(registerID, "") - assert.NoError(t, err) - - b2, err := v.Get(registerID, "") - assert.NoError(t, err) - assert.Nil(t, b2) - }) - - t.Run("ValueInCache", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == registerID { - return flow.RegisterValue("orange"), nil - } - - return nil, nil - }) - - err := v.Set(registerID, "", flow.RegisterValue("apple")) - assert.NoError(t, err) - - b1, err := v.Get(registerID, "") - assert.NoError(t, err) - assert.Equal(t, flow.RegisterValue("apple"), b1) - - err = v.Delete(registerID, "") - assert.NoError(t, err) - - b2, err := v.Get(registerID, "") - assert.NoError(t, err) - assert.Nil(t, b2) - }) -} - func TestViewMergeView(t *testing.T) { - registerID1 := "fruit" - - registerID2 := "vegetable" - - registerID3 := "diary" + registerID1 := flow.NewRegisterID("fruit", "") + registerID2 := flow.NewRegisterID("vegetable", "") + registerID3 := flow.NewRegisterID("diary", "") t.Run("EmptyView", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) chView := v.NewChild() - err := chView.Set(registerID1, "", flow.RegisterValue("apple")) + err := chView.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) - err = chView.Set(registerID2, "", flow.RegisterValue("carrot")) + err = chView.Set(registerID2, flow.RegisterValue("carrot")) assert.NoError(t, err) err = v.MergeView(chView) assert.NoError(t, err) - b1, err := v.Get(registerID1, "") + b1, err := v.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) - b2, err := v.Get(registerID2, "") + b2, err := v.Get(registerID2) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("carrot"), b2) }) t.Run("EmptyDelta", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - err := v.Set(registerID1, "", flow.RegisterValue("apple")) + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) - err = v.Set(registerID2, "", flow.RegisterValue("carrot")) + err = v.Set(registerID2, flow.RegisterValue("carrot")) assert.NoError(t, err) chView := v.NewChild() err = v.MergeView(chView) assert.NoError(t, err) - b1, err := v.Get(registerID1, "") + b1, err := v.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) - b2, err := v.Get(registerID2, "") + b2, err := v.Get(registerID2) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("carrot"), b2) }) t.Run("NoCollisions", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - err := v.Set(registerID1, "", flow.RegisterValue("apple")) + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) chView := v.NewChild() - err = chView.Set(registerID2, "", flow.RegisterValue("carrot")) + err = chView.Set(registerID2, flow.RegisterValue("carrot")) assert.NoError(t, err) err = v.MergeView(chView) assert.NoError(t, err) - b1, err := v.Get(registerID1, "") + b1, err := v.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) - b2, err := v.Get(registerID2, "") + b2, err := v.Get(registerID2) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("carrot"), b2) }) t.Run("OverwriteSetValue", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - err := v.Set(registerID1, "", flow.RegisterValue("apple")) + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) chView := v.NewChild() - err = chView.Set(registerID1, "", flow.RegisterValue("orange")) + err = chView.Set(registerID1, flow.RegisterValue("orange")) assert.NoError(t, err) err = v.MergeView(chView) assert.NoError(t, err) - b, err := v.Get(registerID1, "") + b, err := v.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("orange"), b) }) - t.Run("OverwriteDeletedValue", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + t.Run("OverwriteValue", func(t *testing.T) { + v := delta.NewDeltaView(nil) - err := v.Set(registerID1, "", flow.RegisterValue("apple")) - assert.NoError(t, err) - err = v.Delete(registerID1, "") + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) chView := v.NewChild() - err = chView.Set(registerID1, "", flow.RegisterValue("orange")) + err = chView.Set(registerID1, flow.RegisterValue("orange")) assert.NoError(t, err) err = v.MergeView(chView) assert.NoError(t, err) - b, err := v.Get(registerID1, "") + b, err := v.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("orange"), b) }) - t.Run("DeleteSetValue", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) - - err := v.Set(registerID1, "", flow.RegisterValue("apple")) - assert.NoError(t, err) - - chView := v.NewChild() - err = chView.Delete(registerID1, "") - assert.NoError(t, err) - err = v.MergeView(chView) - assert.NoError(t, err) - - b, err := v.Get(registerID1, "") - assert.NoError(t, err) - assert.Nil(t, b) - }) t.Run("SpockDataMerge", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - register := flow.NewRegisterID("", "") - register.Owner = registerID1 - registerID1Bytes := register.Bytes() - register.Owner = registerID2 - registerID2Bytes := register.Bytes() + registerID1Bytes := registerID1.Bytes() + registerID2Bytes := registerID2.Bytes() expSpock1 := hash.NewSHA3_256() - err := v.Set(registerID1, "", flow.RegisterValue("apple")) + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) hashIt(t, expSpock1, registerID1Bytes) hashIt(t, expSpock1, []byte("apple")) @@ -373,7 +268,7 @@ func TestViewMergeView(t *testing.T) { expSpock2 := hash.NewSHA3_256() chView := v.NewChild() - err = chView.Set(registerID2, "", flow.RegisterValue("carrot")) + err = chView.Set(registerID2, flow.RegisterValue("carrot")) require.NoError(t, err) hashIt(t, expSpock2, registerID2Bytes) hashIt(t, expSpock2, []byte("carrot")) @@ -389,17 +284,15 @@ func TestViewMergeView(t *testing.T) { }) t.Run("RegisterTouchesDataMerge", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - err := v.Set(registerID1, "", flow.RegisterValue("apple")) + err := v.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) chView := v.NewChild() - err = chView.Set(registerID2, "", flow.RegisterValue("carrot")) + err = chView.Set(registerID2, flow.RegisterValue("carrot")) assert.NoError(t, err) - err = chView.Set(registerID3, "", flow.RegisterValue("milk")) + err = chView.Set(registerID3, flow.RegisterValue("milk")) assert.NoError(t, err) err = v.MergeView(chView) @@ -409,26 +302,20 @@ func TestViewMergeView(t *testing.T) { require.Len(t, reads, 3) - r1 := flow.NewRegisterID(registerID1, "") - r2 := flow.NewRegisterID(registerID2, "") - r3 := flow.NewRegisterID(registerID3, "") - assert.Equal(t, map[flow.RegisterID]struct{}{ - r1: struct{}{}, - r2: struct{}{}, - r3: struct{}{}, + registerID1: struct{}{}, + registerID2: struct{}{}, + registerID3: struct{}{}, }, reads) }) } func TestView_RegisterTouches(t *testing.T) { - registerID1 := "fruit" - registerID2 := "vegetable" + registerID1 := flow.NewRegisterID("fruit", "") + registerID2 := flow.NewRegisterID("vegetable", "") - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) t.Run("Empty", func(t *testing.T) { touches := v.Interactions().RegisterTouches() @@ -436,22 +323,15 @@ func TestView_RegisterTouches(t *testing.T) { }) t.Run("Set and Get", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == registerID1 { - return flow.RegisterValue("orange"), nil - } - - if owner == registerID2 { - return flow.RegisterValue("carrot"), nil - } - - return nil, nil - }) - - _, err := v.Get(registerID1, "") + v := delta.NewDeltaView( + testStorage{ + registerID1: "orange", + registerID2: "carrot", + }) + _, err := v.Get(registerID1) assert.NoError(t, err) - err = v.Set(registerID2, "", flow.RegisterValue("apple")) + err = v.Set(registerID2, flow.RegisterValue("apple")) assert.NoError(t, err) touches := v.Interactions().RegisterTouches() @@ -459,91 +339,84 @@ func TestView_RegisterTouches(t *testing.T) { }) } -func TestView_AllRegisters(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) +func TestView_AllRegisterIDs(t *testing.T) { + idA := flow.NewRegisterID("a", "") + idB := flow.NewRegisterID("b", "") + idC := flow.NewRegisterID("c", "") + idD := flow.NewRegisterID("d", "") + idE := flow.NewRegisterID("e", "") + idF := flow.NewRegisterID("f", "") + + v := delta.NewDeltaView(nil) t.Run("Empty", func(t *testing.T) { - regs := v.Interactions().AllRegisters() + regs := v.Interactions().AllRegisterIDs() assert.Empty(t, regs) }) t.Run("Set and Get", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == "a" { - return flow.RegisterValue("a_value"), nil - } - - if owner == "b" { - return flow.RegisterValue("b_value"), nil - } - return nil, nil - }) + v := delta.NewDeltaView( + testStorage{ + idA: "a_value", + idB: "b_value", + }) - _, err := v.Get("a", "") + _, err := v.Get(idA) assert.NoError(t, err) - _, err = v.Get("b", "") + _, err = v.Get(idB) assert.NoError(t, err) - err = v.Set("c", "", flow.RegisterValue("c_value")) + err = v.Set(idC, flow.RegisterValue("c_value")) assert.NoError(t, err) - err = v.Set("d", "", flow.RegisterValue("d_value")) + err = v.Set(idD, flow.RegisterValue("d_value")) assert.NoError(t, err) - err = v.Touch("e", "") + err = v.Set(idE, flow.RegisterValue("e_value")) assert.NoError(t, err) - err = v.Touch("f", "") + err = v.Set(idF, flow.RegisterValue("f_value")) assert.NoError(t, err) - allRegs := v.Interactions().AllRegisters() + allRegs := v.Interactions().AllRegisterIDs() assert.Len(t, allRegs, 6) }) t.Run("With Merge", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if owner == "a" { - return flow.RegisterValue("a_value"), nil - } - - if owner == "b" { - return flow.RegisterValue("b_value"), nil - } - return nil, nil - }) + v := delta.NewDeltaView( + testStorage{ + idA: "a_value", + idB: "b_value", + }) vv := v.NewChild() - _, err := vv.Get("a", "") + _, err := vv.Get(idA) assert.NoError(t, err) - _, err = vv.Get("b", "") + _, err = vv.Get(idB) assert.NoError(t, err) - err = vv.Set("c", "", flow.RegisterValue("c_value")) + err = vv.Set(idC, flow.RegisterValue("c_value")) assert.NoError(t, err) - err = vv.Set("d", "", flow.RegisterValue("d_value")) + err = vv.Set(idD, flow.RegisterValue("d_value")) assert.NoError(t, err) - err = vv.Touch("e", "") + err = vv.Set(idE, flow.RegisterValue("e_value")) assert.NoError(t, err) - err = vv.Touch("f", "") + err = vv.Set(idF, flow.RegisterValue("f_value")) assert.NoError(t, err) err = v.MergeView(vv) assert.NoError(t, err) - allRegs := v.Interactions().AllRegisters() + allRegs := v.Interactions().AllRegisterIDs() assert.Len(t, allRegs, 6) }) } func TestView_Reads(t *testing.T) { - registerID1 := "fruit" - registerID2 := "vegetable" + registerID1 := flow.NewRegisterID("fruit", "") + registerID2 := flow.NewRegisterID("vegetable", "") - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) t.Run("Empty", func(t *testing.T) { reads := v.Interactions().Reads @@ -551,28 +424,23 @@ func TestView_Reads(t *testing.T) { }) t.Run("Set and Get", func(t *testing.T) { - v := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - return nil, nil - }) + v := delta.NewDeltaView(nil) - _, err := v.Get(registerID2, "") + _, err := v.Get(registerID2) assert.NoError(t, err) - _, err = v.Get(registerID1, "") + _, err = v.Get(registerID1) assert.NoError(t, err) - _, err = v.Get(registerID2, "") + _, err = v.Get(registerID2) assert.NoError(t, err) touches := v.Interactions().Reads require.Len(t, touches, 2) - r1 := flow.NewRegisterID(registerID1, "") - r2 := flow.NewRegisterID(registerID2, "") - assert.Equal(t, map[flow.RegisterID]struct{}{ - r1: struct{}{}, - r2: struct{}{}, + registerID1: struct{}{}, + registerID2: struct{}{}, }, touches) }) } diff --git a/engine/execution/state/mock/execution_state.go b/engine/execution/state/mock/execution_state.go index f78954a63ec..680f402ee38 100644 --- a/engine/execution/state/mock/execution_state.go +++ b/engine/execution/state/mock/execution_state.go @@ -5,7 +5,9 @@ package mock import ( context "context" + execution "github.com/onflow/flow-go/engine/execution" delta "github.com/onflow/flow-go/engine/execution/state/delta" + flow "github.com/onflow/flow-go/model/flow" messages "github.com/onflow/flow-go/model/messages" @@ -239,13 +241,13 @@ func (_m *ExecutionState) RetrieveStateDelta(_a0 context.Context, _a1 flow.Ident return r0, r1 } -// SaveExecutionResults provides a mock function with given fields: ctx, header, endState, chunkDataPacks, executionReceipt, events, serviceEvents, results -func (_m *ExecutionState) SaveExecutionResults(ctx context.Context, header *flow.Header, endState flow.StateCommitment, chunkDataPacks []*flow.ChunkDataPack, executionReceipt *flow.ExecutionReceipt, events []flow.EventsList, serviceEvents flow.EventsList, results []flow.TransactionResult) error { - ret := _m.Called(ctx, header, endState, chunkDataPacks, executionReceipt, events, serviceEvents, results) +// SaveExecutionResults provides a mock function with given fields: ctx, result, executionReceipt +func (_m *ExecutionState) SaveExecutionResults(ctx context.Context, result *execution.ComputationResult, executionReceipt *flow.ExecutionReceipt) error { + ret := _m.Called(ctx, result, executionReceipt) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *flow.Header, flow.StateCommitment, []*flow.ChunkDataPack, *flow.ExecutionReceipt, []flow.EventsList, flow.EventsList, []flow.TransactionResult) error); ok { - r0 = rf(ctx, header, endState, chunkDataPacks, executionReceipt, events, serviceEvents, results) + if rf, ok := ret.Get(0).(func(context.Context, *execution.ComputationResult, *flow.ExecutionReceipt) error); ok { + r0 = rf(ctx, result, executionReceipt) } else { r0 = ret.Error(0) } diff --git a/engine/execution/state/state.go b/engine/execution/state/state.go index 4b5e7ea0a48..0569fa1ef27 100644 --- a/engine/execution/state/state.go +++ b/engine/execution/state/state.go @@ -8,6 +8,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/state/delta" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" @@ -68,9 +69,11 @@ type ExecutionState interface { UpdateHighestExecutedBlockIfHigher(context.Context, *flow.Header) error - SaveExecutionResults(ctx context.Context, header *flow.Header, endState flow.StateCommitment, - chunkDataPacks []*flow.ChunkDataPack, - executionReceipt *flow.ExecutionReceipt, events []flow.EventsList, serviceEvents flow.EventsList, results []flow.TransactionResult) error + SaveExecutionResults( + ctx context.Context, + result *execution.ComputationResult, + executionReceipt *flow.ExecutionReceipt, + ) error } const ( @@ -138,9 +141,9 @@ func NewExecutionState( } -func makeSingleValueQuery(commitment flow.StateCommitment, owner, key string) (*ledger.QuerySingleValue, error) { +func makeSingleValueQuery(commitment flow.StateCommitment, id flow.RegisterID) (*ledger.QuerySingleValue, error) { return ledger.NewQuerySingleValue(ledger.State(commitment), - RegisterIDToKey(flow.NewRegisterID(owner, key)), + RegisterIDToKey(id), ) } @@ -169,46 +172,63 @@ func RegisterEntriesToKeysValues( return keys, values } -func LedgerGetRegister(ldg ledger.Ledger, commitment flow.StateCommitment) delta.GetRegisterFunc { +// TODO(patrick): revisit caching. readCache needs to be mutex guarded for +// parallel execution. +type LedgerStorageSnapshot struct { + ledger ledger.Ledger + commitment flow.StateCommitment - readCache := make(map[flow.RegisterID]flow.RegisterEntry) - - return func(owner, key string) (flow.RegisterValue, error) { - regID := flow.RegisterID{ - Owner: owner, - Key: key, - } - - if value, ok := readCache[regID]; ok { - return value.Value, nil - } + readCache map[flow.RegisterID]flow.RegisterValue +} - query, err := makeSingleValueQuery(commitment, owner, key) +func NewLedgerStorageSnapshot( + ldg ledger.Ledger, + commitment flow.StateCommitment, +) delta.StorageSnapshot { + return &LedgerStorageSnapshot{ + ledger: ldg, + commitment: commitment, + readCache: make(map[flow.RegisterID]flow.RegisterValue), + } +} - if err != nil { - return nil, fmt.Errorf("cannot create ledger query: %w", err) - } +func (storage *LedgerStorageSnapshot) Get( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + if value, ok := storage.readCache[id]; ok { + return value, nil + } - value, err := ldg.GetSingleValue(query) + query, err := makeSingleValueQuery(storage.commitment, id) + if err != nil { + return nil, fmt.Errorf("cannot create ledger query: %w", err) + } - if err != nil { - return nil, fmt.Errorf("error getting register (%s) value at %x: %w", key, commitment, err) - } + value, err := storage.ledger.GetSingleValue(query) + if err != nil { + return nil, fmt.Errorf( + "error getting register (%s) value at %x: %w", + id, + storage.commitment, + err) + } - // Prevent caching of value with len zero - if len(value) == 0 { - return nil, nil - } + // Prevent caching of value with len zero + if len(value) == 0 { + return nil, nil + } - // don't cache value with len zero - readCache[regID] = flow.RegisterEntry{Key: regID, Value: value} + // don't cache value with len zero + storage.readCache[id] = value - return value, nil - } + return value, nil } func (s *state) NewView(commitment flow.StateCommitment) *delta.View { - return delta.NewView(LedgerGetRegister(s.ls, commitment)) + return delta.NewDeltaView(NewLedgerStorageSnapshot(s.ls, commitment)) } type RegisterUpdatesHolder interface { @@ -232,13 +252,6 @@ func CommitDelta(ldg ledger.Ledger, ruh RegisterUpdatesHolder, baseState flow.St return flow.StateCommitment(commit), trieUpdate, nil } -//func (s *state) CommitDelta(ctx context.Context, delta delta.Delta, baseState flow.StateCommitment) (flow.StateCommitment, error) { -// span, _ := s.tracer.StartSpanFromContext(ctx, trace.EXECommitDelta) -// defer span.End() -// -// return CommitDelta(s.ls, delta, baseState) -//} - func (s *state) getRegisters(commit flow.StateCommitment, registerIDs []flow.RegisterID) (*ledger.Query, []ledger.Value, error) { query, err := makeQuery(commit, registerIDs) @@ -327,22 +340,20 @@ func (s *state) GetExecutionResultID(ctx context.Context, blockID flow.Identifie return result.ID(), nil } -func (s *state) SaveExecutionResults(ctx context.Context, header *flow.Header, endState flow.StateCommitment, - chunkDataPacks []*flow.ChunkDataPack, executionReceipt *flow.ExecutionReceipt, events []flow.EventsList, serviceEvents flow.EventsList, - results []flow.TransactionResult) error { - return s.saveExecutionResults(ctx, header, endState, chunkDataPacks, executionReceipt, events, serviceEvents, results) -} - -func (s *state) saveExecutionResults(ctx context.Context, header *flow.Header, endState flow.StateCommitment, - chunkDataPacks []*flow.ChunkDataPack, executionReceipt *flow.ExecutionReceipt, events []flow.EventsList, serviceEvents flow.EventsList, - results []flow.TransactionResult) error { - +func (s *state) SaveExecutionResults( + ctx context.Context, + result *execution.ComputationResult, + executionReceipt *flow.ExecutionReceipt, +) error { spew.Config.DisableMethods = true spew.Config.DisablePointerMethods = true - span, childCtx := s.tracer.StartSpanFromContext(ctx, trace.EXEStateSaveExecutionResults) + span, childCtx := s.tracer.StartSpanFromContext( + ctx, + trace.EXEStateSaveExecutionResults) defer span.End() + header := result.ExecutableBlock.Block.Header blockID := header.ID() // Write Batch is BadgerDB feature designed for handling lots of writes @@ -352,34 +363,37 @@ func (s *state) saveExecutionResults(ctx context.Context, header *flow.Header, e // but it's the closest thing to atomicity we could have batch := badgerstorage.NewBatch(s.db) - for _, chunkDataPack := range chunkDataPacks { + for _, chunkDataPack := range result.ChunkDataPacks { err := s.chunkDataPacks.BatchStore(chunkDataPack, batch) if err != nil { return fmt.Errorf("cannot store chunk data pack: %w", err) } - err = s.headers.BatchIndexByChunkID(header.ID(), chunkDataPack.ChunkID, batch) + err = s.headers.BatchIndexByChunkID(blockID, chunkDataPack.ChunkID, batch) if err != nil { return fmt.Errorf("cannot index chunk data pack by blockID: %w", err) } } - err := s.commits.BatchStore(blockID, endState, batch) + err := s.commits.BatchStore(blockID, result.EndState, batch) if err != nil { return fmt.Errorf("cannot store state commitment: %w", err) } - err = s.events.BatchStore(blockID, events, batch) + err = s.events.BatchStore(blockID, result.Events, batch) if err != nil { return fmt.Errorf("cannot store events: %w", err) } - err = s.serviceEvents.BatchStore(blockID, serviceEvents, batch) + err = s.serviceEvents.BatchStore(blockID, result.ServiceEvents, batch) if err != nil { return fmt.Errorf("cannot store service events: %w", err) } - err = s.transactionResults.BatchStore(blockID, results, batch) + err = s.transactionResults.BatchStore( + blockID, + result.TransactionResults, + batch) if err != nil { return fmt.Errorf("cannot store transaction result: %w", err) } diff --git a/engine/execution/state/state_test.go b/engine/execution/state/state_test.go index aeb387208ac..92eb806123c 100644 --- a/engine/execution/state/state_test.go +++ b/engine/execution/state/state_test.go @@ -67,9 +67,9 @@ func prepareTest(f func(t *testing.T, es state.ExecutionState, l *ledger.Ledger) } func TestExecutionStateWithTrieStorage(t *testing.T) { - registerID1 := "fruit" + registerID1 := flow.NewRegisterID("fruit", "") - registerID2 := "vegetable" + registerID2 := flow.NewRegisterID("vegetable", "") t.Run("commit write and read new state", prepareTest(func(t *testing.T, es state.ExecutionState, l *ledger.Ledger) { // TODO: use real block ID @@ -78,9 +78,9 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { view1 := es.NewView(sc1) - err = view1.Set(registerID1, "", flow.RegisterValue("apple")) + err = view1.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) - err = view1.Set(registerID2, "", flow.RegisterValue("carrot")) + err = view1.Set(registerID2, flow.RegisterValue("carrot")) assert.NoError(t, err) sc2, update, err := state.CommitDelta(l, view1.Delta(), sc1) @@ -90,11 +90,19 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { assert.Len(t, update.Paths, 2) assert.Len(t, update.Payloads, 2) - key1 := ledger2.NewKey([]ledger2.KeyPart{ledger2.NewKeyPart(0, []byte(registerID1)), ledger2.NewKeyPart(2, []byte(""))}) + key1 := ledger2.NewKey( + []ledger2.KeyPart{ + ledger2.NewKeyPart(0, []byte(registerID1.Owner)), + ledger2.NewKeyPart(2, []byte(registerID1.Key)), + }) path1, err := pathfinder.KeyToPath(key1, ledger.DefaultPathFinderVersion) assert.NoError(t, err) - key2 := ledger2.NewKey([]ledger2.KeyPart{ledger2.NewKeyPart(0, []byte(registerID2)), ledger2.NewKeyPart(2, []byte(""))}) + key2 := ledger2.NewKey( + []ledger2.KeyPart{ + ledger2.NewKeyPart(0, []byte(registerID2.Owner)), + ledger2.NewKeyPart(2, []byte(registerID2.Key)), + }) path2, err := pathfinder.KeyToPath(key2, ledger.DefaultPathFinderVersion) assert.NoError(t, err) @@ -115,9 +123,9 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { view2 := es.NewView(sc2) - b1, err := view2.Get(registerID1, "") + b1, err := view2.Get(registerID1) assert.NoError(t, err) - b2, err := view2.Get(registerID2, "") + b2, err := view2.Get(registerID2) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) @@ -131,14 +139,14 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { view1 := es.NewView(sc1) - err = view1.Set(registerID1, "", []byte("apple")) + err = view1.Set(registerID1, []byte("apple")) assert.NoError(t, err) sc2, _, err := state.CommitDelta(l, view1.Delta(), sc1) assert.NoError(t, err) // update value and get resulting state commitment view2 := es.NewView(sc2) - err = view2.Set(registerID1, "", []byte("orange")) + err = view2.Set(registerID1, []byte("orange")) assert.NoError(t, err) sc3, _, err := state.CommitDelta(l, view2.Delta(), sc2) @@ -151,10 +159,10 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { view4 := es.NewView(sc3) // fetch the value at both versions - b1, err := view3.Get(registerID1, "") + b1, err := view3.Get(registerID1) assert.NoError(t, err) - b2, err := view4.Get(registerID1, "") + b2, err := view4.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) @@ -168,9 +176,9 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { // set initial value view1 := es.NewView(sc1) - err = view1.Set(registerID1, "", []byte("apple")) + err = view1.Set(registerID1, []byte("apple")) assert.NoError(t, err) - err = view1.Set(registerID2, "", []byte("apple")) + err = view1.Set(registerID2, []byte("apple")) assert.NoError(t, err) sc2, _, err := state.CommitDelta(l, view1.Delta(), sc1) @@ -178,7 +186,7 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { // update value and get resulting state commitment view2 := es.NewView(sc2) - err = view2.Delete(registerID1, "") + err = view2.Set(registerID1, nil) assert.NoError(t, err) sc3, _, err := state.CommitDelta(l, view2.Delta(), sc2) @@ -191,10 +199,10 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { view4 := es.NewView(sc3) // fetch the value at both versions - b1, err := view3.Get(registerID1, "") + b1, err := view3.Get(registerID1) assert.NoError(t, err) - b2, err := view4.Get(registerID1, "") + b2, err := view4.Get(registerID1) assert.NoError(t, err) assert.Equal(t, flow.RegisterValue("apple"), b1) @@ -208,9 +216,9 @@ func TestExecutionStateWithTrieStorage(t *testing.T) { // set initial value view1 := es.NewView(sc1) - err = view1.Set(registerID1, "", flow.RegisterValue("apple")) + err = view1.Set(registerID1, flow.RegisterValue("apple")) assert.NoError(t, err) - err = view1.Set(registerID2, "", flow.RegisterValue("apple")) + err = view1.Set(registerID2, flow.RegisterValue("apple")) assert.NoError(t, err) sc2, _, err := state.CommitDelta(l, view1.Delta(), sc1) diff --git a/engine/execution/state/unittest/fixtures.go b/engine/execution/state/unittest/fixtures.go index f1471da26a2..00335c76df3 100644 --- a/engine/execution/state/unittest/fixtures.go +++ b/engine/execution/state/unittest/fixtures.go @@ -10,7 +10,7 @@ import ( ) func StateInteractionsFixture() *delta.SpockSnapshot { - return delta.NewView(nil).Interactions() + return delta.NewDeltaView(nil).Interactions() } func ComputationResultFixture(collectionsSignerIDs [][]flow.Identifier) *execution.ComputationResult { @@ -24,19 +24,46 @@ func ComputationResultFixture(collectionsSignerIDs [][]flow.Identifier) *executi func ComputationResultForBlockFixture( completeBlock *entity.ExecutableBlock, ) *execution.ComputationResult { - numChunks := len(completeBlock.CompleteCollections) + 1 + collections := completeBlock.Collections() + + numChunks := len(collections) + 1 stateViews := make([]*delta.SpockSnapshot, numChunks) stateCommitments := make([]flow.StateCommitment, numChunks) proofs := make([][]byte, numChunks) events := make([]flow.EventsList, numChunks) eventHashes := make([]flow.Identifier, numChunks) spockHashes := make([]crypto.Signature, numChunks) + chunks := make([]*flow.Chunk, 0, numChunks) + chunkDataPacks := make([]*flow.ChunkDataPack, 0, numChunks) for i := 0; i < numChunks; i++ { stateViews[i] = StateInteractionsFixture() stateCommitments[i] = *completeBlock.StartState proofs[i] = unittest.RandomBytes(6) events[i] = make(flow.EventsList, 0) eventHashes[i] = unittest.IdentifierFixture() + + chunk := flow.NewChunk( + completeBlock.ID(), + i, + *completeBlock.StartState, + 0, + unittest.IdentifierFixture(), + *completeBlock.StartState) + chunks = append(chunks, chunk) + + var collection *flow.Collection + if i < len(collections) { + colStruct := collections[i].Collection() + collection = &colStruct + } + + chunkDataPacks = append( + chunkDataPacks, + flow.NewChunkDataPack( + chunk.ID(), + *completeBlock.StartState, + proofs[i], + collection)) } return &execution.ComputationResult{ TransactionResultIndex: make([]int, numChunks), @@ -47,5 +74,8 @@ func ComputationResultForBlockFixture( Events: events, EventsHashes: eventHashes, SpockSignatures: spockHashes, + Chunks: chunks, + ChunkDataPacks: chunkDataPacks, + EndState: *completeBlock.StartState, } } diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 79d2754c0f1..f7f2515a212 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -271,7 +271,8 @@ func CreateAccountsWithSimpleAddresses( if err != nil { return nil, errors.New("error decoding events") } - addr = flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + addr = flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) break } } @@ -303,7 +304,10 @@ func RootBootstrappedLedger(vm fvm.VM, ctx fvm.Context, additionalOptions ...fvm options..., ) - _ = vm.Run(ctx, bootstrap, view) + err := vm.Run(ctx, bootstrap, view) + if err != nil { + panic(err) + } return view } diff --git a/engine/ghost/client/ghost_client.go b/engine/ghost/client/ghost_client.go index 8b08df80bc1..66e5555b002 100644 --- a/engine/ghost/client/ghost_client.go +++ b/engine/ghost/client/ghost_client.go @@ -7,6 +7,7 @@ import ( "io" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow-go/utils/unittest" @@ -33,7 +34,7 @@ type GhostClient struct { func NewGhostClient(addr string) (*GhostClient, error) { - conn, err := grpc.Dial(addr, grpc.WithInsecure()) //nolint:staticcheck + conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, err } diff --git a/engine/ghost/engine/rpc.go b/engine/ghost/engine/rpc.go index 9044c5b3d8d..889dd77a6f8 100644 --- a/engine/ghost/engine/rpc.go +++ b/engine/ghost/engine/rpc.go @@ -15,13 +15,12 @@ import ( "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/utils/grpcutils" ) // Config defines the configurable options for the gRPC server. type Config struct { ListenAddr string - MaxMsgSize int // In bytes + MaxMsgSize uint // In bytes } // RPC implements a gRPC server for the Ghost node @@ -50,17 +49,13 @@ func New(net network.Network, log zerolog.Logger, me module.Local, state protoco codec := cborcodec.NewCodec() - if config.MaxMsgSize == 0 { - config.MaxMsgSize = grpcutils.DefaultMaxMsgSize - } - eng := &RPC{ log: log, unit: engine.NewUnit(), me: me, server: grpc.NewServer( - grpc.MaxRecvMsgSize(config.MaxMsgSize), - grpc.MaxSendMsgSize(config.MaxMsgSize), + grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxMsgSize)), ), config: config, messages: messages, diff --git a/engine/protocol/handler.go b/engine/protocol/handler.go index 7c03c236ecf..a7b96e0c841 100644 --- a/engine/protocol/handler.go +++ b/engine/protocol/handler.go @@ -143,7 +143,7 @@ func (h *Handler) GetBlockByID( func (h *Handler) blockResponse(block *flow.Block, fullResponse bool) (*access.BlockResponse, error) { signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(block.Header) if err != nil { - return nil, err + return nil, err // the block was retrieved from local storage - so no errors are expected } var msg *entities.Block @@ -163,7 +163,7 @@ func (h *Handler) blockResponse(block *flow.Block, fullResponse bool) (*access.B func (h *Handler) blockHeaderResponse(header *flow.Header) (*access.BlockHeaderResponse, error) { signerIDs, err := h.signerIndicesDecoder.DecodeSignerIDs(header) if err != nil { - return nil, err + return nil, err // the block was retrieved from local storage - so no errors are expected } msg, err := convert.BlockHeaderToMessage(header, signerIDs) diff --git a/engine/testutil/mock/nodes.go b/engine/testutil/mock/nodes.go index 3d9bbe03b8f..ac05dfbad4e 100644 --- a/engine/testutil/mock/nodes.go +++ b/engine/testutil/mock/nodes.go @@ -69,6 +69,7 @@ type GenericNode struct { // context and cancel function used to start/stop components Ctx irrecoverable.SignalerContext Cancel context.CancelFunc + Errs <-chan error Log zerolog.Logger Metrics *metrics.NoopCollector @@ -131,8 +132,13 @@ type CollectionNode struct { EpochManagerEngine *epochmgr.Engine } -func (n CollectionNode) Ready() <-chan struct{} { +func (n CollectionNode) Start(t *testing.T) { + go unittest.FailOnIrrecoverableError(t, n.Ctx.Done(), n.Errs) n.IngestionEngine.Start(n.Ctx) + n.EpochManagerEngine.Start(n.Ctx) +} + +func (n CollectionNode) Ready() <-chan struct{} { return util.AllReady( n.PusherEngine, n.ProviderEngine, @@ -202,6 +208,7 @@ type ExecutionNode struct { ExecutionEngine *ComputerWrap RequestEngine *requester.Engine ReceiptsEngine *executionprovider.Engine + FollowerCore module.HotStuffFollower FollowerEngine *followereng.Engine SyncEngine *synchronization.Engine Compactor *complete.Compactor @@ -221,11 +228,14 @@ func (en ExecutionNode) Ready(ctx context.Context) { // new interface. irctx, _ := irrecoverable.WithSignaler(ctx) en.ReceiptsEngine.Start(irctx) + en.FollowerCore.Start(irctx) + en.FollowerEngine.Start(irctx) <-util.AllReady( en.Ledger, en.ReceiptsEngine, en.IngestionEngine, + en.FollowerCore, en.FollowerEngine, en.RequestEngine, en.SyncEngine, @@ -242,6 +252,7 @@ func (en ExecutionNode) Done(cancelFunc context.CancelFunc) { en.IngestionEngine, en.ReceiptsEngine, en.Ledger, + en.FollowerCore, en.FollowerEngine, en.RequestEngine, en.SyncEngine, diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index a17bcd4792e..673cc38e7af 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -17,13 +17,11 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" - "github.com/onflow/flow-go/network/p2p/cache" - - "github.com/onflow/flow-go/module/mempool/queue" - "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/consensus/hotstuff/committees" mockhotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" "github.com/onflow/flow-go/crypto" @@ -77,6 +75,7 @@ import ( consensusMempools "github.com/onflow/flow-go/module/mempool/consensus" "github.com/onflow/flow-go/module/mempool/epochs" "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/mempool/queue" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" @@ -85,6 +84,7 @@ import ( "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/module/validation" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/stub" "github.com/onflow/flow-go/state/protocol" badgerstate "github.com/onflow/flow-go/state/protocol/badger" @@ -169,11 +169,12 @@ func GenericNodeWithStateFixture(t testing.TB, net := stub.NewNetwork(t, identity.NodeID, hub) parentCtx, cancel := context.WithCancel(context.Background()) - ctx, _ := irrecoverable.WithSignaler(parentCtx) + ctx, errs := irrecoverable.WithSignaler(parentCtx) return testmock.GenericNode{ Ctx: ctx, Cancel: cancel, + Errs: errs, Log: log, Metrics: metrics, Tracer: tracer, @@ -308,7 +309,7 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H ) require.NoError(t, err) - proposalFactory, err := factories.NewProposalEngineFactory( + complianceEngineFactory, err := factories.NewComplianceEngineFactory( node.Log, node.Net, node.Me, @@ -318,12 +319,14 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H ) require.NoError(t, err) + syncCoreFactory, err := factories.NewSyncCoreFactory(node.Log, chainsync.DefaultConfig()) + require.NoError(t, err) + syncFactory, err := factories.NewSyncEngineFactory( node.Log, node.Metrics, node.Net, node.Me, - chainsync.DefaultConfig(), ) require.NoError(t, err) @@ -335,19 +338,30 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H node.Me, node.PublicDB, node.State, + node.Metrics, + node.Metrics, createMetrics, - consensus.WithInitialTimeout(time.Second*2), ) require.NoError(t, err) + messageHubFactory := factories.NewMessageHubFactory( + node.Log, + node.Net, + node.Me, + node.Metrics, + node.State, + ) + factory := factories.NewEpochComponentsFactory( node.Me, pools, builderFactory, clusterStateFactory, hotstuffFactory, - proposalFactory, + complianceEngineFactory, + syncCoreFactory, syncFactory, + messageHubFactory, ) rootQCVoter := new(mockmodule.ClusterRootQCVoter) @@ -624,7 +638,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit Manager: computationEngine, } - syncCore, err := chainsync.New(node.Log, chainsync.DefaultConfig(), metrics.NewChainSyncCollector()) + syncCore, err := chainsync.New(node.Log, chainsync.DefaultConfig(), metrics.NewChainSyncCollector(genesisHead.ChainID), genesisHead.ChainID) require.NoError(t, err) deltas, err := ingestion.NewDeltas(1000) @@ -671,12 +685,15 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit node.ProtocolEvents.AddConsumer(ingestionEngine) followerCore, finalizer := createFollowerCore(t, &node, followerState, finalizationDistributor, rootHead, rootQC) + // mock out hotstuff validator + validator := new(mockhotstuff.Validator) + validator.On("ValidateProposal", mock.Anything).Return(nil) // initialize cleaner for DB cleaner := storage.NewCleaner(node.Log, node.PublicDB, node.Metrics, flow.DefaultValueLogGCFrequency) followerEng, err := follower.New(node.Log, node.Net, node.Me, node.Metrics, node.Metrics, cleaner, - node.Headers, node.Payloads, followerState, pendingBlocks, followerCore, syncCore, node.Tracer) + node.Headers, node.Payloads, followerState, pendingBlocks, followerCore, validator, syncCore, node.Tracer) require.NoError(t, err) finalizedHeader, err := synchronization.NewFinalizedHeaderCache(node.Log, node.State, finalizationDistributor) @@ -708,6 +725,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit GenericNode: node, MutableState: followerState, IngestionEngine: ingestionEngine, + FollowerCore: followerCore, FollowerEngine: followerEng, SyncEngine: syncEngine, ExecutionEngine: computation, @@ -751,14 +769,30 @@ type RoundRobinLeaderSelection struct { me flow.Identifier } -func (s *RoundRobinLeaderSelection) Identities(blockID flow.Identifier) (flow.IdentityList, error) { +var _ hotstuff.Replicas = (*RoundRobinLeaderSelection)(nil) +var _ hotstuff.DynamicCommittee = (*RoundRobinLeaderSelection)(nil) + +func (s *RoundRobinLeaderSelection) IdentitiesByBlock(_ flow.Identifier) (flow.IdentityList, error) { + return s.identities, nil +} + +func (s *RoundRobinLeaderSelection) IdentityByBlock(_ flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { + id, found := s.identities.ByNodeID(participantID) + if !found { + return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) + } + + return id, nil +} + +func (s *RoundRobinLeaderSelection) IdentitiesByEpoch(_ uint64) (flow.IdentityList, error) { return s.identities, nil } -func (s *RoundRobinLeaderSelection) Identity(blockID flow.Identifier, participantID flow.Identifier) (*flow.Identity, error) { +func (s *RoundRobinLeaderSelection) IdentityByEpoch(_ uint64, participantID flow.Identifier) (*flow.Identity, error) { id, found := s.identities.ByNodeID(participantID) if !found { - return nil, fmt.Errorf("not found") + return nil, model.NewInvalidSignerErrorf("unknown participant %x", participantID) } return id, nil } @@ -767,11 +801,19 @@ func (s *RoundRobinLeaderSelection) LeaderForView(view uint64) (flow.Identifier, return s.identities[int(view)%len(s.identities)].NodeID, nil } +func (s *RoundRobinLeaderSelection) QuorumThresholdForView(_ uint64) (uint64, error) { + return committees.WeightThresholdToBuildQC(s.identities.TotalWeight()), nil +} + +func (s *RoundRobinLeaderSelection) TimeoutThresholdForView(_ uint64) (uint64, error) { + return committees.WeightThresholdToTimeout(s.identities.TotalWeight()), nil +} + func (s *RoundRobinLeaderSelection) Self() flow.Identifier { return s.me } -func (s *RoundRobinLeaderSelection) DKG(blockID flow.Identifier) (hotstuff.DKG, error) { +func (s *RoundRobinLeaderSelection) DKG(_ uint64) (hotstuff.DKG, error) { return nil, fmt.Errorf("error") } @@ -788,8 +830,9 @@ func createFollowerCore(t *testing.T, node *testmock.GenericNode, followerState // mock finalization updater verifier := &mockhotstuff.Verifier{} - verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything).Return(nil) - verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything).Return(nil) + verifier.On("VerifyVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + verifier.On("VerifyQC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + verifier.On("VerifyTC", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) finalizer := confinalizer.NewFinalizer(node.PublicDB, node.Headers, followerState, trace.NewNoopTracer()) diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index 93bfc27facd..df56b85af9d 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -261,7 +261,10 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB ) // create state.View - view := delta.NewView(state.LedgerGetRegister(led, startStateCommitment)) + view := delta.NewDeltaView( + state.NewLedgerStorageSnapshot( + led, + startStateCommitment)) committer := committer.NewLedgerViewCommitter(led, trace.NewNoopTracer()) derivedBlockData := derived.NewEmptyDerivedBlockData() diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 08f78dc48b2..506f0abbee3 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/committees" "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" hotsignature "github.com/onflow/flow-go/consensus/hotstuff/signature" + hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" @@ -109,10 +110,11 @@ type FollowerServiceBuilder struct { SyncCore *synchronization.Core FinalizationDistributor *pubsub.FinalizationDistributor FinalizedHeader *synceng.FinalizedHeaderCache - Committee hotstuff.Committee + Committee hotstuff.DynamicCommittee Finalized *flow.Header Pending []*flow.Header FollowerCore module.HotStuffFollower + Validator hotstuff.Validator // for the observer, the sync engine participants provider is the libp2p peer store which is not // available until after the network has started. Hence, a factory function that needs to be called just before // creating the sync engine @@ -171,7 +173,7 @@ func (builder *FollowerServiceBuilder) buildFollowerState() *FollowerServiceBuil func (builder *FollowerServiceBuilder) buildSyncCore() *FollowerServiceBuilder { builder.Module("sync core", func(node *cmd.NodeConfig) error { - syncCore, err := synchronization.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector()) + syncCore, err := synchronization.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) builder.SyncCore = syncCore return err @@ -181,14 +183,15 @@ func (builder *FollowerServiceBuilder) buildSyncCore() *FollowerServiceBuilder { } func (builder *FollowerServiceBuilder) buildCommittee() *FollowerServiceBuilder { - builder.Module("committee", func(node *cmd.NodeConfig) error { + builder.Component("committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { // initialize consensus committee's membership state - // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee + // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS committee // Note: node.Me.NodeID() is not part of the consensus committee committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) + node.ProtocolEvents.AddConsumer(committee) builder.Committee = committee - return err + return committee, err }) return builder @@ -214,6 +217,7 @@ func (builder *FollowerServiceBuilder) buildFollowerCore() *FollowerServiceBuild packer := hotsignature.NewConsensusSigDataPacker(builder.Committee) // initialize the verifier for the protocol consensus verifier := verification.NewCombinedVerifier(builder.Committee, packer) + builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) followerCore, err := consensus.NewFollower(node.Logger, builder.Committee, node.Storage.Headers, final, verifier, builder.FinalizationDistributor, node.RootBlock.Header, node.RootQC, builder.Finalized, builder.Pending) @@ -246,6 +250,7 @@ func (builder *FollowerServiceBuilder) buildFollowerEngine() *FollowerServiceBui builder.FollowerState, conCache, builder.FollowerCore, + builder.Validator, builder.SyncCore, node.Tracer, follower.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold)), @@ -469,7 +474,8 @@ func (builder *FollowerServiceBuilder) InitIDProviders() { // The following wrapper allows to black-list byzantine nodes via an admin command: // the wrapper overrides the 'Ejected' flag of blocked nodes to true - builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB) + builder.NodeBlockListDistributor = cache.NewNodeBlockListDistributor() + builder.IdentityProvider, err = cache.NewNodeBlocklistWrapper(idCache, node.DB, builder.NodeBlockListDistributor) if err != nil { return fmt.Errorf("could not initialize NodeBlocklistWrapper: %w", err) } @@ -704,9 +710,10 @@ func (builder *FollowerServiceBuilder) enqueueConnectWithStakedAN() { // interval, and validators. The network.Middleware is then passed into the initNetwork function. func (builder *FollowerServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, - validators ...network.MessageValidator) network.Middleware { + validators ...network.MessageValidator, +) network.Middleware { slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network) - builder.Middleware = middleware.NewMiddleware( + mw := middleware.NewMiddleware( builder.Logger, libp2pNode, nodeID, @@ -716,7 +723,10 @@ func (builder *FollowerServiceBuilder) initMiddleware(nodeID flow.Identifier, builder.IDTranslator, builder.CodecFactory(), slashingViolationsConsumer, - middleware.WithMessageValidators(validators...)) - + middleware.WithMessageValidators(validators...), + // use default identifier provider + ) + builder.NodeBlockListDistributor.AddConsumer(mw) + builder.Middleware = mw return builder.Middleware } diff --git a/fvm/accounts_test.go b/fvm/accounts_test.go index 73215e2e840..96482580ea6 100644 --- a/fvm/accounts_test.go +++ b/fvm/accounts_test.go @@ -20,6 +20,20 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) +type invalidAccountStatusKeyStorageSnapshot struct{} + +func (invalidAccountStatusKeyStorageSnapshot) Get( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + if id.Key == flow.AccountStatusKey { + return nil, fmt.Errorf("error getting register %s", id) + } + return nil, nil +} + func createAccount( t *testing.T, vm fvm.VM, @@ -50,7 +64,8 @@ func createAccount( data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) return address } @@ -267,7 +282,7 @@ transaction { ?? panic("Unable to borrow reference to administrator resource") } execute { - // Remove account from account creator whitelist. + // Remove account from account creator allowlist. // // Will emit AccountCreatorRemoved(accountCreator: accountCreator). // @@ -287,7 +302,7 @@ transaction { ?? panic("Unable to borrow reference to administrator resource") } execute { - // Add account to account creator whitelist. + // Add account to account creator allowlist. // // Will emit AccountCreatorAdded(accountCreator: accountCreator). // @@ -370,7 +385,8 @@ func TestCreateAccount(t *testing.T) { data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) account, err := vm.GetAccount(ctx, address, view) require.NoError(t, err) @@ -405,7 +421,8 @@ func TestCreateAccount(t *testing.T) { data, err := jsoncdc.Decode(nil, tx.Events[i].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) account, err := vm.GetAccount(ctx, address, view) require.NoError(t, err) @@ -1310,15 +1327,16 @@ func TestAccountBalanceFields(t *testing.T) { } `, address))) - view := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if key == state.AccountStatusKey { - return nil, fmt.Errorf("error getting register %s, %s", flow.BytesToAddress([]byte(owner)).Hex(), key) - } - return nil, nil - }) + view := delta.NewDeltaView( + invalidAccountStatusKeyStorageSnapshot{}) err := vm.Run(ctx, script, view) - require.ErrorContains(t, err, fmt.Sprintf("error getting register %s, %s", address.Hex(), state.AccountStatusKey)) + require.ErrorContains( + t, + err, + fmt.Sprintf( + "error getting register %s", + flow.AccountStatusRegisterID(address))) }), ) @@ -1518,15 +1536,16 @@ func TestGetStorageCapacity(t *testing.T) { } `, address))) - newview := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if key == state.AccountStatusKey { - return nil, fmt.Errorf("error getting register %s, %s", flow.BytesToAddress([]byte(owner)).Hex(), key) - } - return nil, nil - }) + newview := delta.NewDeltaView( + invalidAccountStatusKeyStorageSnapshot{}) err := vm.Run(ctx, script, newview) - require.ErrorContains(t, err, fmt.Sprintf("error getting register %s, %s", address.Hex(), state.AccountStatusKey)) + require.ErrorContains( + t, + err, + fmt.Sprintf( + "error getting register %s", + flow.AccountStatusRegisterID(address))) }), ) } diff --git a/fvm/blocks.go b/fvm/blocks.go deleted file mode 100644 index ea747b7a36f..00000000000 --- a/fvm/blocks.go +++ /dev/null @@ -1,16 +0,0 @@ -package fvm - -import ( - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/storage" -) - -// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/229 -// is merged and integrated. -type Blocks = environment.Blocks - -// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/229 -// is merged and integrated. -func NewBlockFinder(storage storage.Headers) Blocks { - return environment.NewBlockFinder(storage) -} diff --git a/fvm/blueprints/epochs.go b/fvm/blueprints/epochs.go index 1c7cbbeba01..9a60f179854 100644 --- a/fvm/blueprints/epochs.go +++ b/fvm/blueprints/epochs.go @@ -203,20 +203,28 @@ func SetStakingAllowlistTransaction(idTableStakingAddr flow.Address, allowedNode env := templates.Environment{ IDTableAddress: idTableStakingAddr.HexWithPrefix(), } + allowedNodesArg := SetStakingAllowlistTxArg(allowedNodeIDs) + return flow.NewTransactionBody(). + SetScript(templates.GenerateSetApprovedNodesScript(env)). + AddArgument(jsoncdc.MustEncode(allowedNodesArg)). + AddAuthorizer(idTableStakingAddr) +} - cdcNodeIDs := make([]cadence.Value, 0, len(allowedNodeIDs)) +// SetStakingAllowlistTxArg returns the transaction argument for setting the staking allow-list. +func SetStakingAllowlistTxArg(allowedNodeIDs []flow.Identifier) cadence.Value { + cdcDictEntries := make([]cadence.KeyValuePair, 0, len(allowedNodeIDs)) for _, id := range allowedNodeIDs { cdcNodeID, err := cadence.NewString(id.String()) if err != nil { panic(err) } - cdcNodeIDs = append(cdcNodeIDs, cdcNodeID) + kvPair := cadence.KeyValuePair{ + Key: cdcNodeID, + Value: cadence.NewBool(true), + } + cdcDictEntries = append(cdcDictEntries, kvPair) } - - return flow.NewTransactionBody(). - SetScript(templates.GenerateSetApprovedNodesScript(env)). - AddArgument(jsoncdc.MustEncode(cadence.NewArray(cdcNodeIDs))). - AddAuthorizer(idTableStakingAddr) + return cadence.NewDictionary(cdcDictEntries) } // BytesToCadenceArray converts byte slice to cadence array diff --git a/fvm/blueprints/scripts/deployIDTableStakingTransactionTemplate.cdc b/fvm/blueprints/scripts/deployIDTableStakingTransactionTemplate.cdc index ec3eeba3938..e64ea463ece 100644 --- a/fvm/blueprints/scripts/deployIDTableStakingTransactionTemplate.cdc +++ b/fvm/blueprints/scripts/deployIDTableStakingTransactionTemplate.cdc @@ -1,5 +1,7 @@ +// Note: uses a default large candidate limit transaction(code: String, epochTokenPayout: UFix64, rewardCut: UFix64) { prepare(serviceAccount: AuthAccount) { - serviceAccount.contracts.add(name: "FlowIDTableStaking", code: code.decodeHex(), epochTokenPayout: epochTokenPayout, rewardCut: rewardCut) + let candidateNodeLimits: {UInt8: UInt64} = {1: 10000, 2: 10000, 3: 10000, 4: 10000, 5: 10000} + serviceAccount.contracts.add(name: "FlowIDTableStaking", code: code.decodeHex(), epochTokenPayout: epochTokenPayout, rewardCut: rewardCut, candidateNodeLimits: candidateNodeLimits) } } diff --git a/fvm/bootstrap.go b/fvm/bootstrap.go index 391d9ac52fe..77b2991751d 100644 --- a/fvm/bootstrap.go +++ b/fvm/bootstrap.go @@ -349,9 +349,6 @@ func (b *bootstrapExecutor) Execute() error { b.deployIDTableStaking(service, fungibleToken, flowToken, feeContract) - // set the list of nodes which are allowed to stake in this network - b.setStakingAllowlist(service, b.identities.NodeIDs()) - b.deployEpoch(service, fungibleToken, flowToken, feeContract) // deploy staking proxy contract to the service account @@ -365,6 +362,9 @@ func (b *bootstrapExecutor) Execute() error { b.registerNodes(service, fungibleToken, flowToken) + // set the list of nodes which are allowed to stake in this network + b.setStakingAllowlist(service, b.identities.NodeIDs()) + return nil } @@ -891,6 +891,10 @@ func (b *bootstrapExecutor) invokeMetaTransaction( WithTransactionFeesEnabled(false), WithAuthorizationChecksEnabled(false), WithSequenceNumberCheckAndIncrementEnabled(false), + + // disable interaction and computation limits for bootstrapping + WithMemoryAndInteractionLimitsDisabled(), + WithComputationLimit(math.MaxUint64), ) // use new derived transaction data for each meta transaction. @@ -903,7 +907,6 @@ func (b *bootstrapExecutor) invokeMetaTransaction( } err = Run(tx.NewExecutor(ctx, b.txnState, prog)) - txErr, fatalErr := errors.SplitErrorTypes(err) - return txErr, fatalErr + return tx.Err, err } diff --git a/fvm/context.go b/fvm/context.go index b73bd2d4bbb..3d6e168e621 100644 --- a/fvm/context.go +++ b/fvm/context.go @@ -226,7 +226,8 @@ func WithExtensiveTracing() Option { } } -// TODO(patrick): remove after emulator has been updated. +// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/306 +// is merged and integrated. // // WithTransactionProcessors sets the transaction processors for a // virtual machine context. diff --git a/fvm/derived/derived_block_data.go b/fvm/derived/derived_block_data.go index 78ddafc5fbc..61829470662 100644 --- a/fvm/derived/derived_block_data.go +++ b/fvm/derived/derived_block_data.go @@ -7,13 +7,14 @@ import ( "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/model/flow" ) -// ProgramDependencies are the locations of the programs this program depends on. -type ProgramDependencies map[common.Address]struct{} +// ProgramDependencies are the programs' addresses used by this program. +type ProgramDependencies map[flow.Address]struct{} // AddDependency adds the address as a dependency. -func (d ProgramDependencies) AddDependency(address common.Address) { +func (d ProgramDependencies) AddDependency(address flow.Address) { d[address] = struct{}{} } @@ -151,6 +152,13 @@ func (block *DerivedBlockData) GetProgramForTestingOnly( return block.programs.GetForTestingOnly(addressLocation) } +// CachedPrograms returns the number of programs cached. +// Note: this should only be called after calling commit, otherwise +// the count will contain invalidated entries. +func (block *DerivedBlockData) CachedPrograms() int { + return len(block.programs.items) +} + func (transaction *DerivedTransactionData) GetProgram( addressLocation common.AddressLocation, ) ( diff --git a/fvm/derived/table_test.go b/fvm/derived/table_test.go index f17a48ae978..f044ba18d4e 100644 --- a/fvm/derived/table_test.go +++ b/fvm/derived/table_test.go @@ -832,13 +832,13 @@ type testValueComputer struct { func (computer *testValueComputer) Compute( txnState *state.TransactionState, - key string, + key flow.RegisterID, ) ( int, error, ) { computer.called = true - _, err := txnState.Get("addr", key, true) + _, err := txnState.Get(key) if err != nil { return 0, err } @@ -847,9 +847,9 @@ func (computer *testValueComputer) Compute( } func TestTxnDerivedDataGetOrCompute(t *testing.T) { - blockDerivedData := NewEmptyTable[string, int]() + blockDerivedData := NewEmptyTable[flow.RegisterID, int]() - key := "key" + key := flow.NewRegisterID("addr", "key") value := 12345 t.Run("compute value", func(t *testing.T) { @@ -865,7 +865,7 @@ func TestTxnDerivedDataGetOrCompute(t *testing.T) { assert.Equal(t, value, val) assert.True(t, computer.called) - _, ok := view.Ledger.RegisterTouches[flow.RegisterID{Owner: "addr", Key: key}] + _, ok := view.Ledger.RegisterTouches[key] assert.True(t, ok) // Commit to setup the next test. @@ -886,7 +886,7 @@ func TestTxnDerivedDataGetOrCompute(t *testing.T) { assert.Equal(t, value, val) assert.False(t, computer.called) - _, ok := view.Ledger.RegisterTouches[flow.RegisterID{Owner: "addr", Key: key}] + _, ok := view.Ledger.RegisterTouches[key] assert.True(t, ok) }) } diff --git a/fvm/environment/account_creator.go b/fvm/environment/account_creator.go index 89d9fdd23c2..2e303b24d60 100644 --- a/fvm/environment/account_creator.go +++ b/fvm/environment/account_creator.go @@ -3,7 +3,6 @@ package environment import ( "fmt" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/errors" @@ -53,9 +52,9 @@ func NewParseRestrictedAccountCreator( } func (creator ParseRestrictedAccountCreator) CreateAccount( - payer runtime.Address, + payer common.Address, ) ( - runtime.Address, + common.Address, error, ) { return parseRestrict1Arg1Ret( @@ -66,19 +65,19 @@ func (creator ParseRestrictedAccountCreator) CreateAccount( } type AccountCreator interface { - CreateAccount(payer runtime.Address) (runtime.Address, error) + CreateAccount(payer common.Address) (common.Address, error) } type NoAccountCreator struct { } func (NoAccountCreator) CreateAccount( - payer runtime.Address, + payer common.Address, ) ( - runtime.Address, + common.Address, error, ) { - return runtime.Address{}, errors.NewOperationNotSupportedError( + return common.Address{}, errors.NewOperationNotSupportedError( "CreateAccount") } @@ -147,10 +146,7 @@ func NewAccountCreator( } func (creator *accountCreator) bytes() ([]byte, error) { - stateBytes, err := creator.txnState.Get( - "", - state.AddressStateKey, - creator.txnState.EnforceLimits()) + stateBytes, err := creator.txnState.Get(flow.AddressStateRegisterID) if err != nil { return nil, fmt.Errorf( "failed to read address generator state from the state: %w", @@ -195,10 +191,8 @@ func (creator *accountCreator) NextAddress() (flow.Address, error) { // update the ledger state err = creator.txnState.Set( - "", - state.AddressStateKey, - addressGenerator.Bytes(), - creator.txnState.EnforceLimits()) + flow.AddressStateRegisterID, + addressGenerator.Bytes()) if err != nil { return address, fmt.Errorf( "failed to update the state with address generator state: %w", @@ -237,12 +231,12 @@ func (creator *accountCreator) createBasicAccount( ) { flowAddress, err := creator.NextAddress() if err != nil { - return flow.Address{}, err + return flow.EmptyAddress, err } err = creator.accounts.Create(publicKeys, flowAddress) if err != nil { - return flow.Address{}, fmt.Errorf("create account failed: %w", err) + return flow.EmptyAddress, fmt.Errorf("create account failed: %w", err) } return flowAddress, nil @@ -258,9 +252,9 @@ func (creator *accountCreator) CreateBootstrapAccount( } func (creator *accountCreator) CreateAccount( - payer runtime.Address, + payer common.Address, ) ( - runtime.Address, + common.Address, error, ) { defer creator.tracer.StartChildSpan(trace.FVMEnvCreateAccount).End() @@ -271,7 +265,7 @@ func (creator *accountCreator) CreateAccount( } // don't enforce limit during account creation - var addr runtime.Address + var addr common.Address creator.txnState.RunWithAllLimitsDisabled(func() { addr, err = creator.createAccount(payer) }) @@ -280,9 +274,9 @@ func (creator *accountCreator) CreateAccount( } func (creator *accountCreator) createAccount( - payer runtime.Address, + payer common.Address, ) ( - runtime.Address, + common.Address, error, ) { flowAddress, err := creator.createBasicAccount(nil) @@ -300,5 +294,5 @@ func (creator *accountCreator) createAccount( } creator.metrics.RuntimeSetNumberOfAccounts(creator.AddressCount()) - return runtime.Address(flowAddress), nil + return common.Address(flowAddress), nil } diff --git a/fvm/environment/account_creator_test.go b/fvm/environment/account_creator_test.go index 7de8ffaf967..9c2b4b9ffb8 100644 --- a/fvm/environment/account_creator_test.go +++ b/fvm/environment/account_creator_test.go @@ -27,7 +27,7 @@ func Test_NewAccountCreator_GeneratingUpdatesState(t *testing.T) { _, err := creator.NextAddress() require.NoError(t, err) - stateBytes, err := view.Get("", state.AddressStateKey) + stateBytes, err := view.Get(flow.AddressStateRegisterID) require.NoError(t, err) require.Equal(t, flow.BytesToAddress(stateBytes), flow.HexToAddress("01")) @@ -35,7 +35,9 @@ func Test_NewAccountCreator_GeneratingUpdatesState(t *testing.T) { func Test_NewAccountCreator_UsesLedgerState(t *testing.T) { view := utils.NewSimpleView() - err := view.Set("", state.AddressStateKey, flow.HexToAddress("01").Bytes()) + err := view.Set( + flow.AddressStateRegisterID, + flow.HexToAddress("01").Bytes()) require.NoError(t, err) chain := flow.MonotonicEmulator.Chain() @@ -45,7 +47,7 @@ func Test_NewAccountCreator_UsesLedgerState(t *testing.T) { _, err = creator.NextAddress() require.NoError(t, err) - stateBytes, err := view.Get("", state.AddressStateKey) + stateBytes, err := view.Get(flow.AddressStateRegisterID) require.NoError(t, err) require.Equal(t, flow.BytesToAddress(stateBytes), flow.HexToAddress("02")) diff --git a/fvm/environment/account_freezer.go b/fvm/environment/account_freezer.go index 03cdc8750d1..19b21b3db08 100644 --- a/fvm/environment/account_freezer.go +++ b/fvm/environment/account_freezer.go @@ -19,7 +19,7 @@ type AccountFreezer interface { // Note that the script variant will return OperationNotSupportedError. SetAccountFrozen(address common.Address, frozen bool) error - FrozenAccounts() []common.Address + FrozenAccounts() []flow.Address Reset() } @@ -51,7 +51,7 @@ func (freezer ParseRestrictedAccountFreezer) SetAccountFrozen( frozen) } -func (freezer ParseRestrictedAccountFreezer) FrozenAccounts() []common.Address { +func (freezer ParseRestrictedAccountFreezer) FrozenAccounts() []flow.Address { return freezer.impl.FrozenAccounts() } @@ -61,7 +61,7 @@ func (freezer ParseRestrictedAccountFreezer) Reset() { type NoAccountFreezer struct{} -func (NoAccountFreezer) FrozenAccounts() []common.Address { +func (NoAccountFreezer) FrozenAccounts() []flow.Address { return nil } @@ -78,7 +78,7 @@ type accountFreezer struct { accounts Accounts transactionInfo TransactionInfo - frozenAccounts []common.Address + frozenAccounts []flow.Address } func NewAccountFreezer( @@ -99,21 +99,21 @@ func (freezer *accountFreezer) Reset() { freezer.frozenAccounts = nil } -func (freezer *accountFreezer) FrozenAccounts() []common.Address { +func (freezer *accountFreezer) FrozenAccounts() []flow.Address { return freezer.frozenAccounts } func (freezer *accountFreezer) SetAccountFrozen( - address common.Address, + runtimeAddress common.Address, frozen bool, ) error { - flowAddress := flow.Address(address) + address := flow.ConvertAddress(runtimeAddress) - if flowAddress == freezer.serviceAddress { + if address == freezer.serviceAddress { return fmt.Errorf( "setting account frozen failed: %w", errors.NewValueErrorf( - flowAddress.String(), + address.String(), "cannot freeze service account")) } @@ -126,7 +126,7 @@ func (freezer *accountFreezer) SetAccountFrozen( "the service account")) } - err := freezer.accounts.SetAccountFrozen(flowAddress, frozen) + err := freezer.accounts.SetAccountFrozen(address, frozen) if err != nil { return fmt.Errorf("setting account frozen failed: %w", err) } diff --git a/fvm/environment/account_info.go b/fvm/environment/account_info.go index 5d6c820fea8..630ca58d070 100644 --- a/fvm/environment/account_info.go +++ b/fvm/environment/account_info.go @@ -128,7 +128,7 @@ func NewAccountInfo( } func (info *accountInfo) GetStorageUsed( - address common.Address, + runtimeAddress common.Address, ) ( uint64, error, @@ -140,7 +140,8 @@ func (info *accountInfo) GetStorageUsed( return 0, fmt.Errorf("get storage used failed: %w", err) } - value, err := info.accounts.GetStorageUsed(flow.Address(address)) + value, err := info.accounts.GetStorageUsed( + flow.ConvertAddress(runtimeAddress)) if err != nil { return 0, fmt.Errorf("get storage used failed: %w", err) } @@ -157,7 +158,7 @@ func StorageMBUFixToBytesUInt(result cadence.Value) uint64 { } func (info *accountInfo) GetStorageCapacity( - address common.Address, + runtimeAddress common.Address, ) ( uint64, error, @@ -169,7 +170,8 @@ func (info *accountInfo) GetStorageCapacity( return 0, fmt.Errorf("get storage capacity failed: %w", err) } - result, invokeErr := info.systemContracts.AccountStorageCapacity(address) + result, invokeErr := info.systemContracts.AccountStorageCapacity( + runtimeAddress) if invokeErr != nil { return 0, invokeErr } @@ -181,7 +183,7 @@ func (info *accountInfo) GetStorageCapacity( } func (info *accountInfo) GetAccountBalance( - address common.Address, + runtimeAddress common.Address, ) ( uint64, error, @@ -193,7 +195,7 @@ func (info *accountInfo) GetAccountBalance( return 0, fmt.Errorf("get account balance failed: %w", err) } - result, invokeErr := info.systemContracts.AccountBalance(address) + result, invokeErr := info.systemContracts.AccountBalance(runtimeAddress) if invokeErr != nil { return 0, invokeErr } @@ -201,7 +203,7 @@ func (info *accountInfo) GetAccountBalance( } func (info *accountInfo) GetAccountAvailableBalance( - address common.Address, + runtimeAddress common.Address, ) ( uint64, error, @@ -216,7 +218,8 @@ func (info *accountInfo) GetAccountAvailableBalance( return 0, fmt.Errorf("get account available balance failed: %w", err) } - result, invokeErr := info.systemContracts.AccountAvailableBalance(address) + result, invokeErr := info.systemContracts.AccountAvailableBalance( + runtimeAddress) if invokeErr != nil { return 0, invokeErr } diff --git a/fvm/environment/account_key_reader.go b/fvm/environment/account_key_reader.go index 7616d230d3f..2f84708d97b 100644 --- a/fvm/environment/account_key_reader.go +++ b/fvm/environment/account_key_reader.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/crypto" "github.com/onflow/flow-go/fvm/errors" @@ -21,13 +22,13 @@ type AccountKeyReader interface { // the given index. An error is returned if the specified account does not // exist, the provided index is not valid, or if the key retrieval fails. GetAccountKey( - address runtime.Address, + address common.Address, keyIndex int, ) ( *runtime.AccountKey, error, ) - AccountKeysCount(address runtime.Address) (uint64, error) + AccountKeysCount(address common.Address) (uint64, error) } type ParseRestrictedAccountKeyReader struct { @@ -46,7 +47,7 @@ func NewParseRestrictedAccountKeyReader( } func (reader ParseRestrictedAccountKeyReader) GetAccountKey( - address runtime.Address, + address common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -60,7 +61,7 @@ func (reader ParseRestrictedAccountKeyReader) GetAccountKey( keyIndex) } -func (reader ParseRestrictedAccountKeyReader) AccountKeysCount(address runtime.Address) (uint64, error) { +func (reader ParseRestrictedAccountKeyReader) AccountKeysCount(address common.Address) (uint64, error) { return parseRestrict1Arg1Ret( reader.txnState, "AccountKeysCount", @@ -89,7 +90,7 @@ func NewAccountKeyReader( } func (reader *accountKeyReader) GetAccountKey( - address runtime.Address, + runtimeAddress common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -111,11 +112,11 @@ func (reader *accountKeyReader) GetAccountKey( return nil, nil } - accountAddress := flow.Address(address) + address := flow.ConvertAddress(runtimeAddress) // address verification is also done in this step accountPublicKey, err := reader.accounts.GetPublicKey( - accountAddress, + address, uint64(keyIndex)) if err != nil { // If a key is not found at a given index, then return a nil key with @@ -138,7 +139,12 @@ func (reader *accountKeyReader) GetAccountKey( return runtimeAccountKey, nil } -func (reader *accountKeyReader) AccountKeysCount(address runtime.Address) (uint64, error) { +func (reader *accountKeyReader) AccountKeysCount( + runtimeAddress common.Address, +) ( + uint64, + error, +) { defer reader.tracer.StartChildSpan(trace.FVMEnvAccountKeysCount).End() formatErr := func(err error) (uint64, error) { @@ -150,13 +156,17 @@ func (reader *accountKeyReader) AccountKeysCount(address runtime.Address) (uint6 return formatErr(err) } - accountAddress := flow.Address(address) - // address verification is also done in this step - return reader.accounts.GetPublicKeyCount(accountAddress) + return reader.accounts.GetPublicKeyCount( + flow.ConvertAddress(runtimeAddress)) } -func FlowToRuntimeAccountKey(flowKey flow.AccountPublicKey) (*runtime.AccountKey, error) { +func FlowToRuntimeAccountKey( + flowKey flow.AccountPublicKey, +) ( + *runtime.AccountKey, + error, +) { signAlgo := crypto.CryptoToRuntimeSigningAlgorithm(flowKey.SignAlgo) if signAlgo == runtime.SignatureAlgorithmUnknown { return nil, errors.NewValueErrorf( diff --git a/fvm/environment/account_key_updater.go b/fvm/environment/account_key_updater.go index cc795006c3e..862b13d0ff4 100644 --- a/fvm/environment/account_key_updater.go +++ b/fvm/environment/account_key_updater.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/sema" fgcrypto "github.com/onflow/flow-go/crypto" @@ -85,7 +86,7 @@ type AccountKeyUpdater interface { // if the key insertion fails. // // Note that the script variant will return OperationNotSupportedError. - AddEncodedAccountKey(address runtime.Address, publicKey []byte) error + AddEncodedAccountKey(runtimeAddress common.Address, publicKey []byte) error // RevokeEncodedAccountKey revokes a public key by index from an existing // account. @@ -95,7 +96,7 @@ type AccountKeyUpdater interface { // // Note that the script variant will return OperationNotSupportedError. RevokeEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, index int, ) ( []byte, @@ -109,7 +110,7 @@ type AccountKeyUpdater interface { // // Note that the script variant will return OperationNotSupportedError. AddAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey *runtime.PublicKey, hashAlgo runtime.HashAlgorithm, weight int, @@ -128,7 +129,7 @@ type AccountKeyUpdater interface { // // Note that the script variant will return OperationNotSupportedError. RevokeAccountKey( - address runtime.Address, + runtimeAddress common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -152,19 +153,19 @@ func NewParseRestrictedAccountKeyUpdater( } func (updater ParseRestrictedAccountKeyUpdater) AddEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey []byte, ) error { return parseRestrict2Arg( updater.txnState, trace.FVMEnvAddEncodedAccountKey, updater.impl.AddEncodedAccountKey, - address, + runtimeAddress, publicKey) } func (updater ParseRestrictedAccountKeyUpdater) RevokeEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, index int, ) ( []byte, @@ -174,12 +175,12 @@ func (updater ParseRestrictedAccountKeyUpdater) RevokeEncodedAccountKey( updater.txnState, trace.FVMEnvRevokeEncodedAccountKey, updater.impl.RevokeEncodedAccountKey, - address, + runtimeAddress, index) } func (updater ParseRestrictedAccountKeyUpdater) AddAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey *runtime.PublicKey, hashAlgo runtime.HashAlgorithm, weight int, @@ -191,14 +192,14 @@ func (updater ParseRestrictedAccountKeyUpdater) AddAccountKey( updater.txnState, trace.FVMEnvAddAccountKey, updater.impl.AddAccountKey, - address, + runtimeAddress, publicKey, hashAlgo, weight) } func (updater ParseRestrictedAccountKeyUpdater) RevokeAccountKey( - address runtime.Address, + runtimeAddress common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -208,21 +209,21 @@ func (updater ParseRestrictedAccountKeyUpdater) RevokeAccountKey( updater.txnState, trace.FVMEnvRevokeAccountKey, updater.impl.RevokeAccountKey, - address, + runtimeAddress, keyIndex) } type NoAccountKeyUpdater struct{} func (NoAccountKeyUpdater) AddEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey []byte, ) error { return errors.NewOperationNotSupportedError("AddEncodedAccountKey") } func (NoAccountKeyUpdater) RevokeEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, index int, ) ( []byte, @@ -232,7 +233,7 @@ func (NoAccountKeyUpdater) RevokeEncodedAccountKey( } func (NoAccountKeyUpdater) AddAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey *runtime.PublicKey, hashAlgo runtime.HashAlgorithm, weight int, @@ -244,7 +245,7 @@ func (NoAccountKeyUpdater) AddAccountKey( } func (NoAccountKeyUpdater) RevokeAccountKey( - address runtime.Address, + runtimeAddress common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -283,7 +284,7 @@ func NewAccountKeyUpdater( // This function returns an error if the specified account does not exist or // if the key insertion fails. func (updater *accountKeyUpdater) addAccountKey( - address runtime.Address, + address flow.Address, publicKey *runtime.PublicKey, hashAlgo runtime.HashAlgorithm, weight int, @@ -291,19 +292,17 @@ func (updater *accountKeyUpdater) addAccountKey( *runtime.AccountKey, error, ) { - accountAddress := flow.Address(address) - - ok, err := updater.accounts.Exists(accountAddress) + ok, err := updater.accounts.Exists(address) if err != nil { return nil, fmt.Errorf("adding account key failed: %w", err) } if !ok { return nil, fmt.Errorf( "adding account key failed: %w", - errors.NewAccountNotFoundError(accountAddress)) + errors.NewAccountNotFoundError(address)) } - keyIndex, err := updater.accounts.GetPublicKeyCount(accountAddress) + keyIndex, err := updater.accounts.GetPublicKeyCount(address) if err != nil { return nil, fmt.Errorf("adding account key failed: %w", err) } @@ -317,7 +316,7 @@ func (updater *accountKeyUpdater) addAccountKey( return nil, fmt.Errorf("adding account key failed: %w", err) } - err = updater.accounts.AppendPublicKey(accountAddress, *accountPublicKey) + err = updater.accounts.AppendPublicKey(address, *accountPublicKey) if err != nil { return nil, fmt.Errorf("adding account key failed: %w", err) } @@ -341,15 +340,13 @@ func (updater *accountKeyUpdater) addAccountKey( // TODO (ramtin) do we have to return runtime.AccountKey for this method or // can be separated into another method func (updater *accountKeyUpdater) revokeAccountKey( - address runtime.Address, + address flow.Address, keyIndex int, ) ( *runtime.AccountKey, error, ) { - accountAddress := flow.Address(address) - - ok, err := updater.accounts.Exists(accountAddress) + ok, err := updater.accounts.Exists(address) if err != nil { return nil, fmt.Errorf("revoking account key failed: %w", err) } @@ -357,7 +354,7 @@ func (updater *accountKeyUpdater) revokeAccountKey( if !ok { return nil, fmt.Errorf( "revoking account key failed: %w", - errors.NewAccountNotFoundError(accountAddress)) + errors.NewAccountNotFoundError(address)) } // Don't return an error for invalid key indices @@ -367,7 +364,7 @@ func (updater *accountKeyUpdater) revokeAccountKey( var publicKey flow.AccountPublicKey publicKey, err = updater.accounts.GetPublicKey( - accountAddress, + address, uint64(keyIndex)) if err != nil { // If a key is not found at a given index, then return a nil key with @@ -384,7 +381,7 @@ func (updater *accountKeyUpdater) revokeAccountKey( publicKey.Revoked = true _, err = updater.accounts.SetPublicKey( - accountAddress, + address, uint64(keyIndex), publicKey) if err != nil { @@ -429,18 +426,16 @@ func (updater *accountKeyUpdater) revokeAccountKey( // * NewAccountNotFoundError - if the specified account does not exist // * ValueError - if the provided encodedPublicKey is not valid public key func (updater *accountKeyUpdater) InternalAddEncodedAccountKey( - address runtime.Address, + address flow.Address, encodedPublicKey []byte, ) error { - accountAddress := flow.Address(address) - - ok, err := updater.accounts.Exists(accountAddress) + ok, err := updater.accounts.Exists(address) if err != nil { return fmt.Errorf("adding encoded account key failed: %w", err) } if !ok { - return errors.NewAccountNotFoundError(accountAddress) + return errors.NewAccountNotFoundError(address) } var publicKey flow.AccountPublicKey @@ -456,7 +451,7 @@ func (updater *accountKeyUpdater) InternalAddEncodedAccountKey( err)) } - err = updater.accounts.AppendPublicKey(accountAddress, publicKey) + err = updater.accounts.AppendPublicKey(address, publicKey) if err != nil { return fmt.Errorf("adding encoded account key failed: %w", err) } @@ -469,21 +464,19 @@ func (updater *accountKeyUpdater) InternalAddEncodedAccountKey( // This function returns an error if the specified account does not exist, the // provided key is invalid, or if key revoking fails. func (updater *accountKeyUpdater) removeAccountKey( - address runtime.Address, + address flow.Address, keyIndex int, ) ( []byte, error, ) { - accountAddress := flow.Address(address) - - ok, err := updater.accounts.Exists(accountAddress) + ok, err := updater.accounts.Exists(address) if err != nil { return nil, fmt.Errorf("remove account key failed: %w", err) } if !ok { - issue := errors.NewAccountNotFoundError(accountAddress) + issue := errors.NewAccountNotFoundError(address) return nil, fmt.Errorf("remove account key failed: %w", issue) } @@ -496,7 +489,7 @@ func (updater *accountKeyUpdater) removeAccountKey( var publicKey flow.AccountPublicKey publicKey, err = updater.accounts.GetPublicKey( - accountAddress, + address, uint64(keyIndex)) if err != nil { return nil, fmt.Errorf("remove account key failed: %w", err) @@ -506,7 +499,7 @@ func (updater *accountKeyUpdater) removeAccountKey( publicKey.Revoked = true encodedPublicKey, err := updater.accounts.SetPublicKey( - accountAddress, + address, uint64(keyIndex), publicKey) if err != nil { @@ -517,7 +510,7 @@ func (updater *accountKeyUpdater) removeAccountKey( } func (updater *accountKeyUpdater) AddEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey []byte, ) error { defer updater.tracer.StartChildSpan( @@ -530,7 +523,8 @@ func (updater *accountKeyUpdater) AddEncodedAccountKey( return fmt.Errorf("add encoded account key failed: %w", err) } - err = updater.accounts.CheckAccountNotFrozen(flow.Address(address)) + address := flow.ConvertAddress(runtimeAddress) + err = updater.accounts.CheckAccountNotFrozen(address) if err != nil { return fmt.Errorf("add encoded account key failed: %w", err) } @@ -549,7 +543,7 @@ func (updater *accountKeyUpdater) AddEncodedAccountKey( } func (updater *accountKeyUpdater) RevokeEncodedAccountKey( - address runtime.Address, + runtimeAddress common.Address, index int, ) ( []byte, @@ -564,7 +558,8 @@ func (updater *accountKeyUpdater) RevokeEncodedAccountKey( return nil, fmt.Errorf("revoke encoded account key failed: %w", err) } - err = updater.accounts.CheckAccountNotFrozen(flow.Address(address)) + address := flow.ConvertAddress(runtimeAddress) + err = updater.accounts.CheckAccountNotFrozen(address) if err != nil { return nil, fmt.Errorf("revoke encoded account key failed: %w", err) } @@ -578,7 +573,7 @@ func (updater *accountKeyUpdater) RevokeEncodedAccountKey( } func (updater *accountKeyUpdater) AddAccountKey( - address runtime.Address, + runtimeAddress common.Address, publicKey *runtime.PublicKey, hashAlgo runtime.HashAlgorithm, weight int, @@ -596,7 +591,7 @@ func (updater *accountKeyUpdater) AddAccountKey( } accKey, err := updater.addAccountKey( - address, + flow.ConvertAddress(runtimeAddress), publicKey, hashAlgo, weight) @@ -608,7 +603,7 @@ func (updater *accountKeyUpdater) AddAccountKey( } func (updater *accountKeyUpdater) RevokeAccountKey( - address runtime.Address, + runtimeAddress common.Address, keyIndex int, ) ( *runtime.AccountKey, @@ -623,5 +618,7 @@ func (updater *accountKeyUpdater) RevokeAccountKey( return nil, fmt.Errorf("revoke account key failed: %w", err) } - return updater.revokeAccountKey(address, keyIndex) + return updater.revokeAccountKey( + flow.ConvertAddress(runtimeAddress), + keyIndex) } diff --git a/fvm/environment/account_key_updater_test.go b/fvm/environment/account_key_updater_test.go index eb83c578e1b..9d88042bf8c 100644 --- a/fvm/environment/account_key_updater_test.go +++ b/fvm/environment/account_key_updater_test.go @@ -7,7 +7,6 @@ import ( "github.com/fxamacker/cbor/v2" "github.com/onflow/atree" - "github.com/onflow/cadence" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/sema" "github.com/stretchr/testify/assert" @@ -30,7 +29,7 @@ func TestAddEncodedAccountKey_error_handling_produces_valid_utf8(t *testing.T) { nil, nil) - address := cadence.BytesToAddress([]byte{1, 2, 3, 4}) + address := flow.BytesToAddress([]byte{1, 2, 3, 4}) // emulate encoded public key (which comes as a user input) // containing bytes which are invalid UTF8 @@ -43,9 +42,7 @@ func TestAddEncodedAccountKey_error_handling_produces_valid_utf8(t *testing.T) { encodedPublicKey, err := flow.EncodeRuntimeAccountPublicKey(accountPublicKey) require.NoError(t, err) - err = akh.InternalAddEncodedAccountKey( - runtime.Address(address), - encodedPublicKey) + err = akh.InternalAddEncodedAccountKey(address, encodedPublicKey) require.Error(t, err) require.True(t, errors.IsValueError(err)) @@ -211,16 +208,16 @@ func (f FakeAccounts) GetPublicKey(address flow.Address, keyIndex uint64) (flow. func (f FakeAccounts) SetPublicKey(_ flow.Address, _ uint64, _ flow.AccountPublicKey) ([]byte, error) { return nil, nil } -func (f FakeAccounts) GetContractNames(_ flow.Address) ([]string, error) { return nil, nil } -func (f FakeAccounts) GetContract(_ string, _ flow.Address) ([]byte, error) { return nil, nil } -func (f FakeAccounts) ContractExists(_ string, _ flow.Address) (bool, error) { return false, nil } -func (f FakeAccounts) SetContract(_ string, _ flow.Address, _ []byte) error { return nil } -func (f FakeAccounts) DeleteContract(_ string, _ flow.Address) error { return nil } -func (f FakeAccounts) Create(_ []flow.AccountPublicKey, _ flow.Address) error { return nil } -func (f FakeAccounts) GetValue(_ flow.Address, _ string) (flow.RegisterValue, error) { return nil, nil } -func (f FakeAccounts) CheckAccountNotFrozen(_ flow.Address) error { return nil } -func (f FakeAccounts) GetStorageUsed(_ flow.Address) (uint64, error) { return 0, nil } -func (f FakeAccounts) SetValue(_ flow.Address, _ string, _ []byte) error { return nil } +func (f FakeAccounts) GetContractNames(_ flow.Address) ([]string, error) { return nil, nil } +func (f FakeAccounts) GetContract(_ string, _ flow.Address) ([]byte, error) { return nil, nil } +func (f FakeAccounts) ContractExists(_ string, _ flow.Address) (bool, error) { return false, nil } +func (f FakeAccounts) SetContract(_ string, _ flow.Address, _ []byte) error { return nil } +func (f FakeAccounts) DeleteContract(_ string, _ flow.Address) error { return nil } +func (f FakeAccounts) Create(_ []flow.AccountPublicKey, _ flow.Address) error { return nil } +func (f FakeAccounts) GetValue(_ flow.RegisterID) (flow.RegisterValue, error) { return nil, nil } +func (f FakeAccounts) CheckAccountNotFrozen(_ flow.Address) error { return nil } +func (f FakeAccounts) GetStorageUsed(_ flow.Address) (uint64, error) { return 0, nil } +func (f FakeAccounts) SetValue(_ flow.RegisterID, _ []byte) error { return nil } func (f FakeAccounts) AllocateStorageIndex(_ flow.Address) (atree.StorageIndex, error) { return atree.StorageIndex{}, nil } diff --git a/fvm/environment/accounts.go b/fvm/environment/accounts.go index 0b13e9c28d0..62b88fab575 100644 --- a/fvm/environment/accounts.go +++ b/fvm/environment/accounts.go @@ -33,10 +33,10 @@ type Accounts interface { SetContract(contractName string, address flow.Address, contract []byte) error DeleteContract(contractName string, address flow.Address) error Create(publicKeys []flow.AccountPublicKey, newAddress flow.Address) error - GetValue(address flow.Address, key string) (flow.RegisterValue, error) + GetValue(id flow.RegisterID) (flow.RegisterValue, error) CheckAccountNotFrozen(address flow.Address) error GetStorageUsed(address flow.Address) (uint64, error) - SetValue(address flow.Address, key string, value flow.RegisterValue) error + SetValue(id flow.RegisterID, value flow.RegisterValue) error AllocateStorageIndex(address flow.Address) (atree.StorageIndex, error) SetAccountFrozen(address flow.Address, frozen bool) error } @@ -53,7 +53,12 @@ func NewAccounts(txnState *state.TransactionState) *StatefulAccounts { } } -func (a *StatefulAccounts) AllocateStorageIndex(address flow.Address) (atree.StorageIndex, error) { +func (a *StatefulAccounts) AllocateStorageIndex( + address flow.Address, +) ( + atree.StorageIndex, + error, +) { // get status status, err := a.getAccountStatus(address) if err != nil { @@ -65,23 +70,28 @@ func (a *StatefulAccounts) AllocateStorageIndex(address flow.Address) (atree.Sto newIndexBytes := index.Next() // store nil so that the setValue for new allocated slabs would be faster - // and won't do ledger getValue for every new slabs (currently happening to compute storage size changes) + // and won't do ledger getValue for every new slabs (currently happening to + // compute storage size changes) // this way the getValue would load this value from deltas key := atree.SlabIndexToLedgerKey(index) - err = a.txnState.Set( - string(address.Bytes()), - string(key), - []byte{}, - false) + a.txnState.RunWithAllLimitsDisabled(func() { + err = a.txnState.Set( + flow.NewRegisterID(string(address.Bytes()), string(key)), + []byte{}) + }) if err != nil { - return atree.StorageIndex{}, fmt.Errorf("failed to allocate an storage index: %w", err) + return atree.StorageIndex{}, fmt.Errorf( + "failed to allocate an storage index: %w", + err) } // update the storageIndex bytes status.SetStorageIndex(newIndexBytes) err = a.setAccountStatus(address, status) if err != nil { - return atree.StorageIndex{}, fmt.Errorf("failed to allocate an storage index: %w", err) + return atree.StorageIndex{}, fmt.Errorf( + "failed to allocate an storage index: %w", + err) } return index, nil } @@ -127,7 +137,7 @@ func (a *StatefulAccounts) Get(address flow.Address) (*flow.Account, error) { } func (a *StatefulAccounts) Exists(address flow.Address) (bool, error) { - accStatusBytes, err := a.GetValue(address, state.AccountStatusKey) + accStatusBytes, err := a.GetValue(flow.AccountStatusRegisterID(address)) if err != nil { return false, err } @@ -147,7 +157,10 @@ func (a *StatefulAccounts) Exists(address flow.Address) (bool, error) { } // Create account sets all required registers on an address. -func (a *StatefulAccounts) Create(publicKeys []flow.AccountPublicKey, newAddress flow.Address) error { +func (a *StatefulAccounts) Create( + publicKeys []flow.AccountPublicKey, + newAddress flow.Address, +) error { exists, err := a.Exists(newAddress) if err != nil { return fmt.Errorf("failed to create a new account: %w", err) @@ -157,34 +170,54 @@ func (a *StatefulAccounts) Create(publicKeys []flow.AccountPublicKey, newAddress } accountStatus := NewAccountStatus() - accountStatus.SetStorageUsed(uint64(RegisterSize(newAddress, state.AccountStatusKey, accountStatus.ToBytes()))) - - err = a.setAccountStatus(newAddress, accountStatus) + storageUsedByTheStatusItself := uint64(RegisterSize( + flow.AccountStatusRegisterID(newAddress), + accountStatus.ToBytes())) + err = a.setAccountStatusStorageUsed( + newAddress, + accountStatus, + storageUsedByTheStatusItself) if err != nil { return fmt.Errorf("failed to create a new account: %w", err) } + return a.SetAllPublicKeys(newAddress, publicKeys) } -func (a *StatefulAccounts) GetPublicKey(address flow.Address, keyIndex uint64) (flow.AccountPublicKey, error) { - publicKey, err := a.GetValue(address, KeyPublicKey(keyIndex)) +func (a *StatefulAccounts) GetPublicKey( + address flow.Address, + keyIndex uint64, +) ( + flow.AccountPublicKey, + error, +) { + publicKey, err := a.GetValue(flow.PublicKeyRegisterID(address, keyIndex)) if err != nil { return flow.AccountPublicKey{}, err } if len(publicKey) == 0 { - return flow.AccountPublicKey{}, errors.NewAccountPublicKeyNotFoundError(address, keyIndex) + return flow.AccountPublicKey{}, errors.NewAccountPublicKeyNotFoundError( + address, + keyIndex) } decodedPublicKey, err := flow.DecodeAccountPublicKey(publicKey, keyIndex) if err != nil { - return flow.AccountPublicKey{}, fmt.Errorf("failed to decode public key: %w", err) + return flow.AccountPublicKey{}, fmt.Errorf( + "failed to decode public key: %w", + err) } return decodedPublicKey, nil } -func (a *StatefulAccounts) GetPublicKeyCount(address flow.Address) (uint64, error) { +func (a *StatefulAccounts) GetPublicKeyCount( + address flow.Address, +) ( + uint64, + error, +) { status, err := a.getAccountStatus(address) if err != nil { return 0, fmt.Errorf("failed to get public key count: %w", err) @@ -192,25 +225,41 @@ func (a *StatefulAccounts) GetPublicKeyCount(address flow.Address) (uint64, erro return status.PublicKeyCount(), nil } -func (a *StatefulAccounts) setPublicKeyCount(address flow.Address, count uint64) error { +func (a *StatefulAccounts) setPublicKeyCount( + address flow.Address, + count uint64, +) error { status, err := a.getAccountStatus(address) if err != nil { - return fmt.Errorf("failed to set public key count for account (%s): %w", address.String(), err) + return fmt.Errorf( + "failed to set public key count for account (%s): %w", + address.String(), + err) } status.SetPublicKeyCount(count) err = a.setAccountStatus(address, status) if err != nil { - return fmt.Errorf("failed to set public key count for account (%s): %w", address.String(), err) + return fmt.Errorf( + "failed to set public key count for account (%s): %w", + address.String(), + err) } return nil } -func (a *StatefulAccounts) GetPublicKeys(address flow.Address) (publicKeys []flow.AccountPublicKey, err error) { +func (a *StatefulAccounts) GetPublicKeys( + address flow.Address, +) ( + publicKeys []flow.AccountPublicKey, + err error, +) { count, err := a.GetPublicKeyCount(address) if err != nil { - return nil, fmt.Errorf("failed to get public key count of account: %w", err) + return nil, fmt.Errorf( + "failed to get public key count of account: %w", + err) } publicKeys = make([]flow.AccountPublicKey, count) @@ -234,26 +283,40 @@ func (a *StatefulAccounts) SetPublicKey( err = publicKey.Validate() if err != nil { encoded, _ := publicKey.MarshalJSON() - return nil, errors.NewValueErrorf(string(encoded), "invalid public key value: %w", err) + return nil, errors.NewValueErrorf( + string(encoded), + "invalid public key value: %w", + err) } encodedPublicKey, err = flow.EncodeAccountPublicKey(publicKey) if err != nil { encoded, _ := publicKey.MarshalJSON() - return nil, errors.NewValueErrorf(string(encoded), "invalid public key value: %w", err) + return nil, errors.NewValueErrorf( + string(encoded), + "invalid public key value: %w", + err) } - err = a.SetValue(address, KeyPublicKey(keyIndex), encodedPublicKey) + err = a.SetValue( + flow.PublicKeyRegisterID(address, keyIndex), + encodedPublicKey) return encodedPublicKey, err } -func (a *StatefulAccounts) SetAllPublicKeys(address flow.Address, publicKeys []flow.AccountPublicKey) error { +func (a *StatefulAccounts) SetAllPublicKeys( + address flow.Address, + publicKeys []flow.AccountPublicKey, +) error { - count := uint64(len(publicKeys)) // len returns int and this will not exceed uint64 + count := uint64(len(publicKeys)) if count >= MaxPublicKeyCount { - return errors.NewAccountPublicKeyLimitError(address, count, MaxPublicKeyCount) + return errors.NewAccountPublicKeyLimitError( + address, + count, + MaxPublicKeyCount) } for i, publicKey := range publicKeys { @@ -266,14 +329,21 @@ func (a *StatefulAccounts) SetAllPublicKeys(address flow.Address, publicKeys []f return a.setPublicKeyCount(address, count) } -func (a *StatefulAccounts) AppendPublicKey(address flow.Address, publicKey flow.AccountPublicKey) error { +func (a *StatefulAccounts) AppendPublicKey( + address flow.Address, + publicKey flow.AccountPublicKey, +) error { if !IsValidAccountKeyHashAlgo(publicKey.HashAlgo) { - return errors.NewValueErrorf(publicKey.HashAlgo.String(), "hashing algorithm type not found") + return errors.NewValueErrorf( + publicKey.HashAlgo.String(), + "hashing algorithm type not found") } if !IsValidAccountKeySignAlgo(publicKey.SignAlgo) { - return errors.NewValueErrorf(publicKey.SignAlgo.String(), "signature algorithm type not found") + return errors.NewValueErrorf( + publicKey.SignAlgo.String(), + "signature algorithm type not found") } count, err := a.GetPublicKeyCount(address) @@ -282,7 +352,10 @@ func (a *StatefulAccounts) AppendPublicKey(address flow.Address, publicKey flow. } if count >= MaxPublicKeyCount { - return errors.NewAccountPublicKeyLimitError(address, count+1, MaxPublicKeyCount) + return errors.NewAccountPublicKeyLimitError( + address, + count+1, + MaxPublicKeyCount) } _, err = a.SetPublicKey(address, count, publicKey) @@ -311,13 +384,14 @@ func IsValidAccountKeyHashAlgo(algo hash.HashingAlgorithm) bool { } } -func ContractKey(contractName string) string { - return state.CodeKeyPrefix + contractName -} - -func (a *StatefulAccounts) getContract(contractName string, address flow.Address) ([]byte, error) { - - contract, err := a.GetValue(address, ContractKey(contractName)) +func (a *StatefulAccounts) getContract( + contractName string, + address flow.Address, +) ( + []byte, + error, +) { + contract, err := a.GetValue(flow.ContractRegisterID(address, contractName)) if err != nil { return nil, err } @@ -325,7 +399,11 @@ func (a *StatefulAccounts) getContract(contractName string, address flow.Address return contract, nil } -func (a *StatefulAccounts) setContract(contractName string, address flow.Address, contract []byte) error { +func (a *StatefulAccounts) setContract( + contractName string, + address flow.Address, + contract []byte, +) error { ok, err := a.Exists(address) if err != nil { return err @@ -335,8 +413,8 @@ func (a *StatefulAccounts) setContract(contractName string, address flow.Address return errors.NewAccountNotFoundError(address) } - var prevContract []byte - prevContract, err = a.GetValue(address, ContractKey(contractName)) + id := flow.ContractRegisterID(address, contractName) + prevContract, err := a.GetValue(id) if err != nil { return errors.NewContractNotFoundError(address, contractName) } @@ -346,7 +424,7 @@ func (a *StatefulAccounts) setContract(contractName string, address flow.Address return nil } - err = a.SetValue(address, ContractKey(contractName), contract) + err = a.SetValue(id, contract) if err != nil { return err } @@ -354,7 +432,10 @@ func (a *StatefulAccounts) setContract(contractName string, address flow.Address return nil } -func (a *StatefulAccounts) setContractNames(contractNames contractNames, address flow.Address) error { +func (a *StatefulAccounts) setContractNames( + contractNames contractNames, + address flow.Address, +) error { ok, err := a.Exists(address) if err != nil { return err @@ -374,8 +455,8 @@ func (a *StatefulAccounts) setContractNames(contractNames contractNames, address } newContractNames := buf.Bytes() - var prevContractNames []byte - prevContractNames, err = a.GetValue(address, state.ContractNamesKey) + id := flow.ContractNamesRegisterID(address) + prevContractNames, err := a.GetValue(id) if err != nil { return fmt.Errorf("cannot retrieve current contract names: %w", err) } @@ -385,11 +466,16 @@ func (a *StatefulAccounts) setContractNames(contractNames contractNames, address return nil } - return a.SetValue(address, state.ContractNamesKey, newContractNames) + return a.SetValue(id, newContractNames) } // GetStorageUsed returns the amount of storage used in bytes by this account -func (a *StatefulAccounts) GetStorageUsed(address flow.Address) (uint64, error) { +func (a *StatefulAccounts) GetStorageUsed( + address flow.Address, +) ( + uint64, + error, +) { status, err := a.getAccountStatus(address) if err != nil { return 0, fmt.Errorf("failed to get storage used: %w", err) @@ -397,59 +483,74 @@ func (a *StatefulAccounts) GetStorageUsed(address flow.Address) (uint64, error) return status.StorageUsed(), nil } -func (a *StatefulAccounts) setStorageUsed(address flow.Address, used uint64) error { +func (a *StatefulAccounts) setStorageUsed( + address flow.Address, + used uint64, +) error { status, err := a.getAccountStatus(address) if err != nil { return fmt.Errorf("failed to set storage used: %w", err) } - status.SetStorageUsed(used) + return a.setAccountStatusStorageUsed(address, status, used) +} - err = a.setAccountStatus(address, status) +func (a *StatefulAccounts) setAccountStatusStorageUsed( + address flow.Address, + status *AccountStatus, + newUsed uint64, +) error { + status.SetStorageUsed(newUsed) + + err := a.setAccountStatus(address, status) if err != nil { return fmt.Errorf("failed to set storage used: %w", err) } return nil } -func (a *StatefulAccounts) GetValue(address flow.Address, key string) (flow.RegisterValue, error) { - return a.txnState.Get( - string(address.Bytes()), - key, - a.txnState.EnforceLimits()) +func (a *StatefulAccounts) GetValue( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + return a.txnState.Get(id) } // SetValue sets a value in address' storage -func (a *StatefulAccounts) SetValue(address flow.Address, key string, value flow.RegisterValue) error { - err := a.updateRegisterSizeChange(address, key, value) +func (a *StatefulAccounts) SetValue( + id flow.RegisterID, + value flow.RegisterValue, +) error { + err := a.updateRegisterSizeChange(id, value) if err != nil { - return fmt.Errorf("failed to update storage used by key %s on account %s: %w", state.PrintableKey(key), address, err) + return fmt.Errorf("failed to update storage for %s: %w", id, err) } - return a.txnState.Set( - string(address.Bytes()), - key, - value, - a.txnState.EnforceLimits()) - + return a.txnState.Set(id, value) } -func (a *StatefulAccounts) updateRegisterSizeChange(address flow.Address, key string, value flow.RegisterValue) error { - if key == state.AccountStatusKey { +func (a *StatefulAccounts) updateRegisterSizeChange( + id flow.RegisterID, + value flow.RegisterValue, +) error { + if id.Key == flow.AccountStatusKey { // size of this register is always fixed size // don't double check this to save time and prevent recursion return nil } - oldValue, err := a.GetValue(address, key) + oldValue, err := a.GetValue(id) if err != nil { return err } - sizeChange := int64(RegisterSize(address, key, value) - RegisterSize(address, key, oldValue)) + sizeChange := int64(RegisterSize(id, value) - RegisterSize(id, oldValue)) if sizeChange == 0 { // register size has not changed. Nothing to do return nil } + address := flow.BytesToAddress([]byte(id.Owner)) oldSize, err := a.GetStorageUsed(address) if err != nil { return err @@ -461,7 +562,7 @@ func (a *StatefulAccounts) updateRegisterSizeChange(address flow.Address, key st absChange := uint64(-sizeChange) if absChange > oldSize { // should never happen - return fmt.Errorf("storage used by key %s on account %s would be negative", state.PrintableKey(key), address.Hex()) + return fmt.Errorf("storage would be negative for %s", id) } newSize = oldSize - absChange } else { @@ -470,11 +571,12 @@ func (a *StatefulAccounts) updateRegisterSizeChange(address flow.Address, key st } // this puts us back in the setValue method. - // The difference is that storage_used update exits early from this function so there isn't any recursion. + // The difference is that storage_used update exits early from this + // function so there isn't any recursion. return a.setStorageUsed(address, newSize) } -func RegisterSize(address flow.Address, key string, value flow.RegisterValue) int { +func RegisterSize(id flow.RegisterID, value flow.RegisterValue) int { if len(value) == 0 { // registers with empty value won't (or don't) exist when stored return 0 @@ -482,39 +584,31 @@ func RegisterSize(address flow.Address, key string, value flow.RegisterValue) in size := 0 // additional 2 is for len prefixes when encoding is happening // we might get rid of these 2s in the future - size += 2 + len(string(address.Bytes())) - size += 2 + len(key) + size += 2 + len(id.Owner) + size += 2 + len(id.Key) size += len(value) return size } -// TODO replace with touch -// TODO handle errors -func (a *StatefulAccounts) touch(address flow.Address, key string) { - _, _ = a.txnState.Get( - string(address.Bytes()), - key, - a.txnState.EnforceLimits()) -} - -func (a *StatefulAccounts) TouchContract(contractName string, address flow.Address) { - contractNames, err := a.getContractNames(address) - if err != nil { - panic(err) - } - if contractNames.Has(contractName) { - a.touch(address, ContractKey(contractName)) - } -} - -// GetContractNames gets a sorted list of names of contracts deployed on an address -func (a *StatefulAccounts) GetContractNames(address flow.Address) ([]string, error) { +// GetContractNames gets a sorted list of names of contracts deployed on an +// address +func (a *StatefulAccounts) GetContractNames( + address flow.Address, +) ( + []string, + error, +) { return a.getContractNames(address) } -func (a *StatefulAccounts) getContractNames(address flow.Address) (contractNames, error) { +func (a *StatefulAccounts) getContractNames( + address flow.Address, +) ( + contractNames, + error, +) { // TODO return fatal error if can't fetch - encContractNames, err := a.GetValue(address, state.ContractNamesKey) + encContractNames, err := a.GetValue(flow.ContractNamesRegisterID(address)) if err != nil { return nil, fmt.Errorf("cannot get deployed contract names: %w", err) } @@ -524,13 +618,22 @@ func (a *StatefulAccounts) getContractNames(address flow.Address) (contractNames cborDecoder := cbor.NewDecoder(buf) err = cborDecoder.Decode(&identifiers) if err != nil { - return nil, fmt.Errorf("cannot decode deployed contract names %x: %w", encContractNames, err) + return nil, fmt.Errorf( + "cannot decode deployed contract names %x: %w", + encContractNames, + err) } } return identifiers, nil } -func (a *StatefulAccounts) ContractExists(contractName string, address flow.Address) (bool, error) { +func (a *StatefulAccounts) ContractExists( + contractName string, + address flow.Address, +) ( + bool, + error, +) { contractNames, err := a.getContractNames(address) if err != nil { return false, err @@ -538,9 +641,16 @@ func (a *StatefulAccounts) ContractExists(contractName string, address flow.Addr return contractNames.Has(contractName), nil } -func (a *StatefulAccounts) GetContract(contractName string, address flow.Address) ([]byte, error) { - // we optimized the happy case here, we look up for the content of the contract - // and if its not there we check if contract exists or if this is another problem. +func (a *StatefulAccounts) GetContract( + contractName string, + address flow.Address, +) ( + []byte, + error, +) { + // we optimized the happy case here, we look up for the content of the + // contract and if its not there we check if contract exists or if this is + // another problem. code, err := a.getContract(contractName, address) if err != nil || len(code) == 0 { exists, err := a.ContractExists(contractName, address) @@ -554,7 +664,11 @@ func (a *StatefulAccounts) GetContract(contractName string, address flow.Address return code, err } -func (a *StatefulAccounts) SetContract(contractName string, address flow.Address, contract []byte) error { +func (a *StatefulAccounts) SetContract( + contractName string, + address flow.Address, + contract []byte, +) error { contractNames, err := a.getContractNames(address) if err != nil { return err @@ -567,7 +681,10 @@ func (a *StatefulAccounts) SetContract(contractName string, address flow.Address return a.setContractNames(contractNames, address) } -func (a *StatefulAccounts) DeleteContract(contractName string, address flow.Address) error { +func (a *StatefulAccounts) DeleteContract( + contractName string, + address flow.Address, +) error { contractNames, err := a.getContractNames(address) if err != nil { return err @@ -583,10 +700,19 @@ func (a *StatefulAccounts) DeleteContract(contractName string, address flow.Addr return a.setContractNames(contractNames, address) } -func (a *StatefulAccounts) getAccountStatus(address flow.Address) (*AccountStatus, error) { - statusBytes, err := a.GetValue(address, state.AccountStatusKey) - if err != nil { - return nil, fmt.Errorf("failed to load account status for the account (%s): %w", address.String(), err) +func (a *StatefulAccounts) getAccountStatus( + address flow.Address, +) ( + *AccountStatus, + error, +) { + id := flow.AccountStatusRegisterID(address) + statusBytes, err := a.GetValue(id) + if err != nil { + return nil, fmt.Errorf( + "failed to load account status for the account (%s): %w", + address.String(), + err) } if len(statusBytes) == 0 { return nil, errors.NewAccountNotFoundError(address) @@ -594,15 +720,27 @@ func (a *StatefulAccounts) getAccountStatus(address flow.Address) (*AccountStatu return AccountStatusFromBytes(statusBytes) } -func (a *StatefulAccounts) setAccountStatus(address flow.Address, status *AccountStatus) error { - err := a.SetValue(address, state.AccountStatusKey, status.ToBytes()) +func (a *StatefulAccounts) setAccountStatus( + address flow.Address, + status *AccountStatus, +) error { + id := flow.AccountStatusRegisterID(address) + err := a.SetValue(id, status.ToBytes()) if err != nil { - return fmt.Errorf("failed to store the account status for account (%s): %w", address.String(), err) + return fmt.Errorf( + "failed to store the account status for account (%s): %w", + address.String(), + err) } return nil } -func (a *StatefulAccounts) GetAccountFrozen(address flow.Address) (bool, error) { +func (a *StatefulAccounts) GetAccountFrozen( + address flow.Address, +) ( + bool, + error, +) { status, err := a.getAccountStatus(address) if err != nil { return false, err @@ -610,7 +748,10 @@ func (a *StatefulAccounts) GetAccountFrozen(address flow.Address) (bool, error) return status.IsAccountFrozen(), nil } -func (a *StatefulAccounts) SetAccountFrozen(address flow.Address, frozen bool) error { +func (a *StatefulAccounts) SetAccountFrozen( + address flow.Address, + frozen bool, +) error { status, err := a.getAccountStatus(address) if err != nil { return err @@ -631,8 +772,9 @@ func (a *StatefulAccounts) CheckAccountNotFrozen(address flow.Address) error { return nil } -// contractNames container for a list of contract names. Should always be sorted. -// To ensure this, don't sort while reading it from storage, but sort it while adding/removing elements +// contractNames container for a list of contract names. Should always be +// sorted. To ensure this, don't sort while reading it from storage, but sort +// it while adding/removing elements type contractNames []string func (l contractNames) Has(contractNames string) bool { @@ -657,7 +799,3 @@ func (l *contractNames) remove(contractName string) { } *l = append((*l)[:i], (*l)[i+1:]...) } - -func KeyPublicKey(index uint64) string { - return fmt.Sprintf("public_key_%d", index) -} diff --git a/fvm/environment/accounts_test.go b/fvm/environment/accounts_test.go index d22a92ca56c..358557fb208 100644 --- a/fvm/environment/accounts_test.go +++ b/fvm/environment/accounts_test.go @@ -68,8 +68,7 @@ func TestAccounts_GetPublicKey(t *testing.T) { view := utils.NewSimpleView() err := view.Set( - string(address.Bytes()), - "public_key_0", + flow.NewRegisterID(string(address.Bytes()), "public_key_0"), ledgerValue, ) require.NoError(t, err) @@ -96,8 +95,7 @@ func TestAccounts_GetPublicKeyCount(t *testing.T) { view := utils.NewSimpleView() err := view.Set( - string(address.Bytes()), - "public_key_count", + flow.NewRegisterID(string(address.Bytes()), "public_key_count"), ledgerValue, ) require.NoError(t, err) @@ -125,8 +123,7 @@ func TestAccounts_GetPublicKeys(t *testing.T) { view := utils.NewSimpleView() err := view.Set( - string(address.Bytes()), - "public_key_count", + flow.NewRegisterID(string(address.Bytes()), "public_key_count"), ledgerValue, ) require.NoError(t, err) @@ -144,29 +141,6 @@ func TestAccounts_GetPublicKeys(t *testing.T) { }) } -// Some old account could be created without key count register -// we recreate it in a test -func TestAccounts_GetWithNoKeysCounter(t *testing.T) { - view := utils.NewSimpleView() - - txnState := state.NewTransactionState(view, state.DefaultParameters()) - accounts := environment.NewAccounts(txnState) - address := flow.HexToAddress("01") - - err := accounts.Create(nil, address) - require.NoError(t, err) - - err = view.Delete( - string(address.Bytes()), - "public_key_count") - - require.NoError(t, err) - - require.NotPanics(t, func() { - _, _ = accounts.Get(address) - }) -} - func TestAccounts_SetContracts(t *testing.T) { address := flow.HexToAddress("0x01") @@ -281,11 +255,12 @@ func TestAccount_StorageUsed(t *testing.T) { txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") + key := flow.NewRegisterID(string(address.Bytes()), "some_key") err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) @@ -298,13 +273,14 @@ func TestAccount_StorageUsed(t *testing.T) { txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") + key := flow.NewRegisterID(string(address.Bytes()), "some_key") err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) @@ -317,13 +293,14 @@ func TestAccount_StorageUsed(t *testing.T) { txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") + key := flow.NewRegisterID(string(address.Bytes()), "some_key") err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(13)) + err = accounts.SetValue(key, createByteArray(13)) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) @@ -336,13 +313,14 @@ func TestAccount_StorageUsed(t *testing.T) { txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") + key := flow.NewRegisterID(string(address.Bytes()), "some_key") err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(11)) + err = accounts.SetValue(key, createByteArray(11)) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) @@ -355,13 +333,14 @@ func TestAccount_StorageUsed(t *testing.T) { txnState := state.NewTransactionState(view, state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") + key := flow.NewRegisterID(string(address.Bytes()), "some_key") err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + err = accounts.SetValue(key, createByteArray(12)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", nil) + err = accounts.SetValue(key, nil) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) @@ -378,19 +357,22 @@ func TestAccount_StorageUsed(t *testing.T) { err := accounts.Create(nil, address) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(12)) + key1 := flow.NewRegisterID(string(address.Bytes()), "some_key") + err = accounts.SetValue(key1, createByteArray(12)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key", createByteArray(11)) + err = accounts.SetValue(key1, createByteArray(11)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key2", createByteArray(22)) + key2 := flow.NewRegisterID(string(address.Bytes()), "some_key2") + err = accounts.SetValue(key2, createByteArray(22)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key2", createByteArray(23)) + err = accounts.SetValue(key2, createByteArray(23)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key3", createByteArray(22)) + key3 := flow.NewRegisterID(string(address.Bytes()), "some_key3") + err = accounts.SetValue(key3, createByteArray(22)) require.NoError(t, err) - err = accounts.SetValue(address, "some_key3", createByteArray(0)) + err = accounts.SetValue(key3, createByteArray(0)) require.NoError(t, err) storageUsed, err := accounts.GetStorageUsed(address) diff --git a/fvm/environment/contract_reader.go b/fvm/environment/contract_reader.go index 1c7329b1784..887b97ec6bf 100644 --- a/fvm/environment/contract_reader.go +++ b/fvm/environment/contract_reader.go @@ -34,7 +34,7 @@ func NewContractReader( } func (reader *ContractReader) GetAccountContractNames( - address runtime.Address, + runtimeAddress common.Address, ) ( []string, error, @@ -49,16 +49,16 @@ func (reader *ContractReader) GetAccountContractNames( return nil, fmt.Errorf("get account contract names failed: %w", err) } - a := flow.Address(address) + address := flow.ConvertAddress(runtimeAddress) - freezeError := reader.accounts.CheckAccountNotFrozen(a) + freezeError := reader.accounts.CheckAccountNotFrozen(address) if freezeError != nil { return nil, fmt.Errorf( "get account contract names failed: %w", freezeError) } - return reader.accounts.GetContractNames(a) + return reader.accounts.GetContractNames(address) } func (reader *ContractReader) ResolveLocation( @@ -94,7 +94,7 @@ func (reader *ContractReader) ResolveLocation( // and no specific identifiers where requested in the import statement, // then fetch all identifiers at this address if len(identifiers) == 0 { - address := flow.Address(addressLocation.Address) + address := flow.ConvertAddress(addressLocation.Address) err := reader.accounts.CheckAccountNotFrozen(address) if err != nil { @@ -140,8 +140,9 @@ func (reader *ContractReader) ResolveLocation( return resolvedLocations, nil } -func (reader *ContractReader) GetCode( - location runtime.Location, +func (reader *ContractReader) getCode( + address flow.Address, + contractName string, ) ( []byte, error, @@ -153,21 +154,12 @@ func (reader *ContractReader) GetCode( return nil, fmt.Errorf("get code failed: %w", err) } - contractLocation, ok := location.(common.AddressLocation) - if !ok { - return nil, errors.NewInvalidLocationErrorf( - location, - "expecting an AddressLocation, but other location types are passed") - } - - address := flow.Address(contractLocation.Address) - err = reader.accounts.CheckAccountNotFrozen(address) if err != nil { return nil, fmt.Errorf("get code failed: %w", err) } - add, err := reader.accounts.GetContract(contractLocation.Name, address) + add, err := reader.accounts.GetContract(contractName, address) if err != nil { return nil, fmt.Errorf("get code failed: %w", err) } @@ -175,27 +167,44 @@ func (reader *ContractReader) GetCode( return add, nil } +func (reader *ContractReader) GetCode( + location runtime.Location, +) ( + []byte, + error, +) { + contractLocation, ok := location.(common.AddressLocation) + if !ok { + return nil, errors.NewInvalidLocationErrorf( + location, + "expecting an AddressLocation, but other location types are passed") + } + + return reader.getCode( + flow.ConvertAddress(contractLocation.Address), + contractLocation.Name) +} + func (reader *ContractReader) GetAccountContractCode( - address runtime.Address, + runtimeAddress common.Address, name string, ) ( - code []byte, - err error, + []byte, + error, ) { defer reader.tracer.StartChildSpan( trace.FVMEnvGetAccountContractCode).End() - err = reader.meter.MeterComputation( + err := reader.meter.MeterComputation( ComputationKindGetAccountContractCode, 1) if err != nil { return nil, fmt.Errorf("get account contract code failed: %w", err) } - code, err = reader.GetCode(common.AddressLocation{ - Address: address, - Name: name, - }) + code, err := reader.getCode( + flow.ConvertAddress(runtimeAddress), + name) if err != nil { return nil, fmt.Errorf("get account contract code failed: %w", err) } diff --git a/fvm/environment/contract_updater.go b/fvm/environment/contract_updater.go index f6adcf0fa3e..13d732c3617 100644 --- a/fvm/environment/contract_updater.go +++ b/fvm/environment/contract_updater.go @@ -6,7 +6,6 @@ import ( "sort" "github.com/onflow/cadence" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/blueprints" @@ -69,14 +68,14 @@ type ContractUpdater interface { // Cadence's runtime API. Note that the script variant will return // OperationNotSupportedError. UpdateAccountContractCode( - address runtime.Address, + address common.Address, name string, code []byte, ) error // Cadence's runtime API. Note that the script variant will return // OperationNotSupportedError. - RemoveAccountContractCode(address runtime.Address, name string) error + RemoveAccountContractCode(address common.Address, name string) error Commit() ([]ContractUpdateKey, error) @@ -99,7 +98,7 @@ func NewParseRestrictedContractUpdater( } func (updater ParseRestrictedContractUpdater) UpdateAccountContractCode( - address runtime.Address, + address common.Address, name string, code []byte, ) error { @@ -113,7 +112,7 @@ func (updater ParseRestrictedContractUpdater) UpdateAccountContractCode( } func (updater ParseRestrictedContractUpdater) RemoveAccountContractCode( - address runtime.Address, + address common.Address, name string, ) error { return parseRestrict2Arg( @@ -138,7 +137,7 @@ func (updater ParseRestrictedContractUpdater) Reset() { type NoContractUpdater struct{} func (NoContractUpdater) UpdateAccountContractCode( - address runtime.Address, + address common.Address, name string, code []byte, ) error { @@ -146,7 +145,7 @@ func (NoContractUpdater) UpdateAccountContractCode( } func (NoContractUpdater) RemoveAccountContractCode( - address runtime.Address, + address common.Address, name string, ) error { return errors.NewOperationNotSupportedError("RemoveAccountContractCode") @@ -164,9 +163,9 @@ type ContractUpdaterStubs interface { RestrictedDeploymentEnabled() bool RestrictedRemovalEnabled() bool - GetAuthorizedAccounts(path cadence.Path) []common.Address + GetAuthorizedAccounts(path cadence.Path) []flow.Address - UseContractAuditVoucher(address runtime.Address, code []byte) (bool, error) + UseContractAuditVoucher(address flow.Address, code []byte) (bool, error) } type contractUpdaterStubsImpl struct { @@ -197,7 +196,7 @@ func (impl *contractUpdaterStubsImpl) getIsContractDeploymentRestricted() ( restricted bool, defined bool, ) { - service := runtime.Address(impl.chain.ServiceAddress()) + service := common.Address(impl.chain.ServiceAddress()) runtime := impl.runtime.BorrowCadenceRuntime() defer impl.runtime.ReturnCadenceRuntime(runtime) @@ -240,15 +239,15 @@ func (impl *contractUpdaterStubsImpl) RestrictedRemovalEnabled() bool { // service account be authorized). func (impl *contractUpdaterStubsImpl) GetAuthorizedAccounts( path cadence.Path, -) []common.Address { +) []flow.Address { // set default to service account only - service := runtime.Address(impl.chain.ServiceAddress()) - defaultAccounts := []runtime.Address{service} + service := impl.chain.ServiceAddress() + defaultAccounts := []flow.Address{service} runtime := impl.runtime.BorrowCadenceRuntime() defer impl.runtime.ReturnCadenceRuntime(runtime) - value, err := runtime.ReadStored(service, path) + value, err := runtime.ReadStored(common.Address(service), path) const warningMsg = "failed to read contract authorized accounts from " + "service account. using default behaviour instead." @@ -266,7 +265,7 @@ func (impl *contractUpdaterStubsImpl) GetAuthorizedAccounts( } func (impl *contractUpdaterStubsImpl) UseContractAuditVoucher( - address runtime.Address, + address flow.Address, code []byte, ) ( bool, @@ -281,7 +280,7 @@ type ContractUpdaterImpl struct { tracer tracing.TracerSpan meter Meter accounts Accounts - transactionInfo TransactionInfo + signingAccounts []flow.Address draftUpdates map[ContractUpdateKey]ContractUpdate @@ -312,7 +311,7 @@ func NewContractUpdater( tracer tracing.TracerSpan, meter Meter, accounts Accounts, - transactionInfo TransactionInfo, + signingAccounts []flow.Address, chain flow.Chain, params ContractUpdaterParams, logger *ProgramLogger, @@ -323,7 +322,7 @@ func NewContractUpdater( tracer: tracer, meter: meter, accounts: accounts, - transactionInfo: transactionInfo, + signingAccounts: signingAccounts, ContractUpdaterStubs: &contractUpdaterStubsImpl{ logger: logger, chain: chain, @@ -338,7 +337,7 @@ func NewContractUpdater( } func (updater *ContractUpdaterImpl) UpdateAccountContractCode( - address runtime.Address, + runtimeAddress common.Address, name string, code []byte, ) error { @@ -352,7 +351,8 @@ func (updater *ContractUpdaterImpl) UpdateAccountContractCode( return fmt.Errorf("update account contract code failed: %w", err) } - err = updater.accounts.CheckAccountNotFrozen(flow.Address(address)) + address := flow.ConvertAddress(runtimeAddress) + err = updater.accounts.CheckAccountNotFrozen(address) if err != nil { return fmt.Errorf("update account contract code failed: %w", err) } @@ -361,7 +361,7 @@ func (updater *ContractUpdaterImpl) UpdateAccountContractCode( address, name, code, - updater.transactionInfo.SigningAccounts()) + updater.signingAccounts) if err != nil { return fmt.Errorf("updating account contract code failed: %w", err) } @@ -370,7 +370,7 @@ func (updater *ContractUpdaterImpl) UpdateAccountContractCode( } func (updater *ContractUpdaterImpl) RemoveAccountContractCode( - address runtime.Address, + runtimeAddress common.Address, name string, ) error { defer updater.tracer.StartChildSpan( @@ -383,7 +383,8 @@ func (updater *ContractUpdaterImpl) RemoveAccountContractCode( return fmt.Errorf("remove account contract code failed: %w", err) } - err = updater.accounts.CheckAccountNotFrozen(flow.Address(address)) + address := flow.ConvertAddress(runtimeAddress) + err = updater.accounts.CheckAccountNotFrozen(address) if err != nil { return fmt.Errorf("remove account contract code failed: %w", err) } @@ -391,7 +392,7 @@ func (updater *ContractUpdaterImpl) RemoveAccountContractCode( err = updater.RemoveContract( address, name, - updater.transactionInfo.SigningAccounts()) + updater.signingAccounts) if err != nil { return fmt.Errorf("remove account contract code failed: %w", err) } @@ -400,21 +401,16 @@ func (updater *ContractUpdaterImpl) RemoveAccountContractCode( } func (updater *ContractUpdaterImpl) SetContract( - address runtime.Address, + address flow.Address, name string, code []byte, - signingAccounts []runtime.Address, -) (err error) { - - flowAddress := flow.Address(address) - + signingAccounts []flow.Address, +) error { // Initial contract deployments must be authorized by signing accounts, // or there must be an audit voucher available. // // Contract updates are always allowed. - - var exists bool - exists, err = updater.accounts.ContractExists(name, flowAddress) + exists, err := updater.accounts.ContractExists(name, address) if err != nil { return err } @@ -453,9 +449,9 @@ func (updater *ContractUpdaterImpl) SetContract( } func (updater *ContractUpdaterImpl) RemoveContract( - address runtime.Address, + address flow.Address, name string, - signingAccounts []runtime.Address, + signingAccounts []flow.Address, ) (err error) { // check if authorized if !updater.isAuthorizedForRemoval(signingAccounts) { @@ -480,12 +476,12 @@ func (updater *ContractUpdaterImpl) Commit() ([]ContractUpdateKey, error) { var err error for _, v := range updateList { if len(v.Code) > 0 { - err = updater.accounts.SetContract(v.Name, flow.BytesToAddress(v.Address.Bytes()), v.Code) + err = updater.accounts.SetContract(v.Name, v.Address, v.Code) if err != nil { return nil, err } } else { - err = updater.accounts.DeleteContract(v.Name, flow.BytesToAddress(v.Address.Bytes())) + err = updater.accounts.DeleteContract(v.Name, v.Address) if err != nil { return nil, err } @@ -522,7 +518,7 @@ func (updater *ContractUpdaterImpl) updates() ( } func (updater *ContractUpdaterImpl) isAuthorizedForDeployment( - signingAccounts []runtime.Address, + signingAccounts []flow.Address, ) bool { if updater.RestrictedDeploymentEnabled() { return updater.isAuthorized( @@ -533,7 +529,7 @@ func (updater *ContractUpdaterImpl) isAuthorizedForDeployment( } func (updater *ContractUpdaterImpl) isAuthorizedForRemoval( - signingAccounts []runtime.Address, + signingAccounts []flow.Address, ) bool { if updater.RestrictedRemovalEnabled() { return updater.isAuthorized( @@ -544,7 +540,7 @@ func (updater *ContractUpdaterImpl) isAuthorizedForRemoval( } func (updater *ContractUpdaterImpl) isAuthorized( - signingAccounts []runtime.Address, + signingAccounts []flow.Address, path cadence.Path, ) bool { accts := updater.GetAuthorizedAccounts(path) diff --git a/fvm/environment/contract_updater_test.go b/fvm/environment/contract_updater_test.go index bb528e345fe..9aabdbdc155 100644 --- a/fvm/environment/contract_updater_test.go +++ b/fvm/environment/contract_updater_test.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/onflow/cadence" - "github.com/onflow/cadence/runtime" - "github.com/onflow/cadence/runtime/common" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -21,10 +19,10 @@ import ( type testContractUpdaterStubs struct { deploymentEnabled bool removalEnabled bool - deploymentAuthorized []common.Address - removalAuthorized []common.Address + deploymentAuthorized []flow.Address + removalAuthorized []flow.Address - auditFunc func(address runtime.Address, code []byte) (bool, error) + auditFunc func(address flow.Address, code []byte) (bool, error) } func (p testContractUpdaterStubs) RestrictedDeploymentEnabled() bool { @@ -37,7 +35,7 @@ func (p testContractUpdaterStubs) RestrictedRemovalEnabled() bool { func (p testContractUpdaterStubs) GetAuthorizedAccounts( path cadence.Path, -) []common.Address { +) []flow.Address { if path == blueprints.ContractDeploymentAuthorizedAddressesPath { return p.deploymentAuthorized } @@ -45,7 +43,7 @@ func (p testContractUpdaterStubs) GetAuthorizedAccounts( } func (p testContractUpdaterStubs) UseContractAuditVoucher( - address runtime.Address, + address flow.Address, code []byte, ) ( bool, @@ -60,7 +58,6 @@ func TestContract_ChildMergeFunctionality(t *testing.T) { state.DefaultParameters()) accounts := environment.NewAccounts(txnState) address := flow.HexToAddress("01") - rAdd := runtime.Address(address) err := accounts.Create(nil, address) require.NoError(t, err) @@ -74,7 +71,11 @@ func TestContract_ChildMergeFunctionality(t *testing.T) { require.Equal(t, len(names), 0) // set contract no need for signing accounts - err = contractUpdater.SetContract(rAdd, "testContract", []byte("ABC"), nil) + err = contractUpdater.SetContract( + address, + "testContract", + []byte("ABC"), + nil) require.NoError(t, err) require.True(t, contractUpdater.HasUpdates()) @@ -91,7 +92,11 @@ func TestContract_ChildMergeFunctionality(t *testing.T) { require.Equal(t, cont, []byte("ABC")) // rollback - err = contractUpdater.SetContract(rAdd, "testContract2", []byte("ABC"), nil) + err = contractUpdater.SetContract( + address, + "testContract2", + []byte("ABC"), + nil) require.NoError(t, err) contractUpdater.Reset() require.False(t, contractUpdater.HasUpdates()) @@ -109,7 +114,7 @@ func TestContract_ChildMergeFunctionality(t *testing.T) { require.Equal(t, cont, []byte("ABC")) // remove - err = contractUpdater.RemoveContract(rAdd, "testContract", nil) + err = contractUpdater.RemoveContract(address, "testContract", nil) require.NoError(t, err) // contract still there because no commit yet @@ -134,22 +139,18 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { accounts := environment.NewAccounts(txnState) authAdd := flow.HexToAddress("01") - rAdd := runtime.Address(authAdd) err := accounts.Create(nil, authAdd) require.NoError(t, err) authRemove := flow.HexToAddress("02") - rRemove := runtime.Address(authRemove) err = accounts.Create(nil, authRemove) require.NoError(t, err) authBoth := flow.HexToAddress("03") - rBoth := runtime.Address(authBoth) err = accounts.Create(nil, authBoth) require.NoError(t, err) unAuth := flow.HexToAddress("04") - unAuthR := runtime.Address(unAuth) err = accounts.Create(nil, unAuth) require.NoError(t, err) @@ -159,9 +160,9 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { testContractUpdaterStubs{ deploymentEnabled: true, removalEnabled: true, - deploymentAuthorized: []common.Address{rAdd, rBoth}, - removalAuthorized: []common.Address{rRemove, rBoth}, - auditFunc: func(address runtime.Address, code []byte) (bool, error) { return false, nil }, + deploymentAuthorized: []flow.Address{authAdd, authBoth}, + removalAuthorized: []flow.Address{authRemove, authBoth}, + auditFunc: func(address flow.Address, code []byte) (bool, error) { return false, nil }, }) } @@ -169,7 +170,11 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("try to set contract with unauthorized account", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{unAuthR}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{unAuth}) require.Error(t, err) require.False(t, contractUpdater.HasUpdates()) }) @@ -177,7 +182,11 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("try to set contract with account only authorized for removal", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{rRemove}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{authRemove}) require.Error(t, err) require.False(t, contractUpdater.HasUpdates()) }) @@ -185,7 +194,11 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("set contract with account authorized for adding", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract2", []byte("ABC"), []common.Address{rAdd}) + err = contractUpdater.SetContract( + authAdd, + "testContract2", + []byte("ABC"), + []flow.Address{authAdd}) require.NoError(t, err) require.True(t, contractUpdater.HasUpdates()) }) @@ -193,7 +206,11 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("set contract with account authorized for adding and removing", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract2", []byte("ABC"), []common.Address{rBoth}) + err = contractUpdater.SetContract( + authAdd, + "testContract2", + []byte("ABC"), + []flow.Address{authBoth}) require.NoError(t, err) require.True(t, contractUpdater.HasUpdates()) }) @@ -201,12 +218,19 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("try to remove contract with unauthorized account", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{rAdd}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{authAdd}) require.NoError(t, err) _, err = contractUpdater.Commit() require.NoError(t, err) - err = contractUpdater.RemoveContract(unAuthR, "testContract2", []common.Address{unAuthR}) + err = contractUpdater.RemoveContract( + unAuth, + "testContract2", + []flow.Address{unAuth}) require.Error(t, err) require.False(t, contractUpdater.HasUpdates()) }) @@ -214,12 +238,19 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("remove contract account authorized for removal", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{rAdd}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{authAdd}) require.NoError(t, err) _, err = contractUpdater.Commit() require.NoError(t, err) - err = contractUpdater.RemoveContract(rRemove, "testContract2", []common.Address{rRemove}) + err = contractUpdater.RemoveContract( + authRemove, + "testContract2", + []flow.Address{authRemove}) require.NoError(t, err) require.True(t, contractUpdater.HasUpdates()) }) @@ -227,12 +258,19 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("try to remove contract with account only authorized for adding", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{rAdd}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{authAdd}) require.NoError(t, err) _, err = contractUpdater.Commit() require.NoError(t, err) - err = contractUpdater.RemoveContract(rAdd, "testContract2", []common.Address{rAdd}) + err = contractUpdater.RemoveContract( + authAdd, + "testContract2", + []flow.Address{authAdd}) require.Error(t, err) require.False(t, contractUpdater.HasUpdates()) }) @@ -240,12 +278,19 @@ func TestContract_AuthorizationFunctionality(t *testing.T) { t.Run("remove contract with account authorized for adding and removing", func(t *testing.T) { contractUpdater := makeUpdater() - err = contractUpdater.SetContract(rAdd, "testContract1", []byte("ABC"), []common.Address{rAdd}) + err = contractUpdater.SetContract( + authAdd, + "testContract1", + []byte("ABC"), + []flow.Address{authAdd}) require.NoError(t, err) _, err = contractUpdater.Commit() require.NoError(t, err) - err = contractUpdater.RemoveContract(rBoth, "testContract2", []common.Address{rBoth}) + err = contractUpdater.RemoveContract( + authBoth, + "testContract2", + []flow.Address{authBoth}) require.NoError(t, err) require.True(t, contractUpdater.HasUpdates()) }) @@ -259,12 +304,10 @@ func TestContract_DeploymentVouchers(t *testing.T) { accounts := environment.NewAccounts(txnState) addressWithVoucher := flow.HexToAddress("01") - addressWithVoucherRuntime := runtime.Address(addressWithVoucher) err := accounts.Create(nil, addressWithVoucher) require.NoError(t, err) addressNoVoucher := flow.HexToAddress("02") - addressNoVoucherRuntime := runtime.Address(addressNoVoucher) err = accounts.Create(nil, addressNoVoucher) require.NoError(t, err) @@ -273,7 +316,7 @@ func TestContract_DeploymentVouchers(t *testing.T) { testContractUpdaterStubs{ deploymentEnabled: true, removalEnabled: true, - auditFunc: func(address runtime.Address, code []byte) (bool, error) { + auditFunc: func(address flow.Address, code []byte) (bool, error) { if address.String() == addressWithVoucher.String() { return true, nil } @@ -283,11 +326,11 @@ func TestContract_DeploymentVouchers(t *testing.T) { // set contract without voucher err = contractUpdater.SetContract( - addressNoVoucherRuntime, + addressNoVoucher, "TestContract1", []byte("pub contract TestContract1 {}"), - []common.Address{ - addressNoVoucherRuntime, + []flow.Address{ + addressNoVoucher, }, ) require.Error(t, err) @@ -295,11 +338,11 @@ func TestContract_DeploymentVouchers(t *testing.T) { // try to set contract with voucher err = contractUpdater.SetContract( - addressWithVoucherRuntime, + addressWithVoucher, "TestContract2", []byte("pub contract TestContract2 {}"), - []common.Address{ - addressWithVoucherRuntime, + []flow.Address{ + addressWithVoucher, }, ) require.NoError(t, err) @@ -314,8 +357,6 @@ func TestContract_ContractUpdate(t *testing.T) { accounts := environment.NewAccounts(txnState) flowAddress := flow.HexToAddress("01") - flowCommonAddress := common.MustBytesToAddress(flowAddress.Bytes()) - runtimeAddress := runtime.Address(flowAddress) err := accounts.Create(nil, flowAddress) require.NoError(t, err) @@ -326,7 +367,7 @@ func TestContract_ContractUpdate(t *testing.T) { testContractUpdaterStubs{ deploymentEnabled: true, removalEnabled: true, - auditFunc: func(address runtime.Address, code []byte) (bool, error) { + auditFunc: func(address flow.Address, code []byte) (bool, error) { // Ensure the voucher check is only called once, // for the initial contract deployment, // and not for the subsequent update @@ -338,11 +379,11 @@ func TestContract_ContractUpdate(t *testing.T) { // deploy contract with voucher err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -354,7 +395,7 @@ func TestContract_ContractUpdate(t *testing.T) { t, []environment.ContractUpdateKey{ { - Address: flowCommonAddress, + Address: flowAddress, Name: "TestContract", }, }, @@ -363,11 +404,11 @@ func TestContract_ContractUpdate(t *testing.T) { // try to update contract without voucher err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -389,8 +430,8 @@ func TestContract_DeterministicErrorOnCommit(t *testing.T) { mockAccounts, testContractUpdaterStubs{}) - address1 := runtime.Address(flow.HexToAddress("0000000000000001")) - address2 := runtime.Address(flow.HexToAddress("0000000000000002")) + address1 := flow.HexToAddress("0000000000000001") + address2 := flow.HexToAddress("0000000000000002") err := contractUpdater.SetContract(address2, "A", []byte("ABC"), nil) require.NoError(t, err) @@ -413,8 +454,6 @@ func TestContract_ContractRemoval(t *testing.T) { accounts := environment.NewAccounts(txnState) flowAddress := flow.HexToAddress("01") - flowCommonAddress := common.MustBytesToAddress(flowAddress.Bytes()) - runtimeAddress := runtime.Address(flowAddress) err := accounts.Create(nil, flowAddress) require.NoError(t, err) @@ -425,7 +464,7 @@ func TestContract_ContractRemoval(t *testing.T) { accounts, testContractUpdaterStubs{ removalEnabled: true, - auditFunc: func(address runtime.Address, code []byte) (bool, error) { + auditFunc: func(address flow.Address, code []byte) (bool, error) { // Ensure the voucher check is only called once, // for the initial contract deployment, // and not for the subsequent update @@ -437,11 +476,11 @@ func TestContract_ContractRemoval(t *testing.T) { // deploy contract with voucher err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -453,7 +492,7 @@ func TestContract_ContractRemoval(t *testing.T) { t, []environment.ContractUpdateKey{ { - Address: flowCommonAddress, + Address: flowAddress, Name: "TestContract", }, }, @@ -462,11 +501,11 @@ func TestContract_ContractRemoval(t *testing.T) { // update should work err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -474,10 +513,10 @@ func TestContract_ContractRemoval(t *testing.T) { // try remove contract should fail err = contractUpdater.RemoveContract( - runtimeAddress, + flowAddress, "TestContract", - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.Error(t, err) @@ -489,7 +528,7 @@ func TestContract_ContractRemoval(t *testing.T) { contractUpdater := environment.NewContractUpdaterForTesting( accounts, testContractUpdaterStubs{ - auditFunc: func(address runtime.Address, code []byte) (bool, error) { + auditFunc: func(address flow.Address, code []byte) (bool, error) { // Ensure the voucher check is only called once, // for the initial contract deployment, // and not for the subsequent update @@ -501,11 +540,11 @@ func TestContract_ContractRemoval(t *testing.T) { // deploy contract with voucher err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -517,7 +556,7 @@ func TestContract_ContractRemoval(t *testing.T) { t, []environment.ContractUpdateKey{ { - Address: flowCommonAddress, + Address: flowAddress, Name: "TestContract", }, }, @@ -526,11 +565,11 @@ func TestContract_ContractRemoval(t *testing.T) { // update should work err = contractUpdater.SetContract( - runtimeAddress, + flowAddress, "TestContract", []byte("pub contract TestContract {}"), - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) @@ -538,10 +577,10 @@ func TestContract_ContractRemoval(t *testing.T) { // try remove contract should fail err = contractUpdater.RemoveContract( - runtimeAddress, + flowAddress, "TestContract", - []common.Address{ - runtimeAddress, + []flow.Address{ + flowAddress, }, ) require.NoError(t, err) diff --git a/fvm/environment/derived_data_invalidator.go b/fvm/environment/derived_data_invalidator.go index 8db4b7007c3..2383971c464 100644 --- a/fvm/environment/derived_data_invalidator.go +++ b/fvm/environment/derived_data_invalidator.go @@ -5,10 +5,11 @@ import ( "github.com/onflow/flow-go/fvm/derived" "github.com/onflow/flow-go/fvm/state" + "github.com/onflow/flow-go/model/flow" ) type ContractUpdateKey struct { - Address common.Address + Address flow.Address Name string } @@ -19,7 +20,7 @@ type ContractUpdate struct { type DerivedDataInvalidator struct { ContractUpdateKeys []ContractUpdateKey - FrozenAccounts []common.Address + FrozenAccounts []flow.Address MeterParamOverridesUpdated bool } @@ -38,12 +39,10 @@ func NewDerivedDataInvalidator( } func meterParamOverridesUpdated(env *facadeEnvironment) bool { - // TODO(patrick): expose lighter weight api to list updated ids only - updatedRegisters := env.txnState.UpdatedRegisters() - serviceAccount := string(env.chain.ServiceAddress().Bytes()) storageDomain := common.PathDomainStorage.Identifier() - for _, registerId := range updatedRegisters.IDs() { + + for _, registerId := range env.txnState.UpdatedRegisterIDs() { // The meter param override values are stored in the service account. if registerId.Owner != serviceAccount { continue @@ -61,8 +60,7 @@ func meterParamOverridesUpdated(env *facadeEnvironment) bool { // // The meter param overrides use storageDomain as input, so any // changes to it must also invalidate the values. - if registerId.Key == storageDomain || - state.IsSlabIndex(registerId.Key) { + if registerId.Key == storageDomain || registerId.IsSlabIndex() { return true } } diff --git a/fvm/environment/derived_data_invalidator_test.go b/fvm/environment/derived_data_invalidator_test.go index 62d621379d3..4343a4c8ff9 100644 --- a/fvm/environment/derived_data_invalidator_test.go +++ b/fvm/environment/derived_data_invalidator_test.go @@ -32,8 +32,8 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { programALoc := common.AddressLocation{Address: cAddressA, Name: "A"} programA := &derived.Program{ Program: nil, - Dependencies: map[common.Address]struct{}{ - cAddressA: {}, + Dependencies: map[flow.Address]struct{}{ + addressA: {}, }, } @@ -42,9 +42,9 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { programBLoc := common.AddressLocation{Address: cAddressB, Name: "B"} programB := &derived.Program{ Program: nil, - Dependencies: map[common.Address]struct{}{ - cAddressA: {}, - cAddressB: {}, + Dependencies: map[flow.Address]struct{}{ + addressA: {}, + addressB: {}, }, } @@ -53,8 +53,8 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { programDLoc := common.AddressLocation{Address: cAddressD, Name: "D"} programD := &derived.Program{ Program: nil, - Dependencies: map[common.Address]struct{}{ - cAddressD: {}, + Dependencies: map[flow.Address]struct{}{ + addressD: {}, }, } @@ -63,12 +63,12 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { programCLoc := common.AddressLocation{Address: cAddressC, Name: "C"} programC := &derived.Program{ Program: nil, - Dependencies: map[common.Address]struct{}{ + Dependencies: map[flow.Address]struct{}{ // C indirectly depends on A trough B - cAddressA: {}, - cAddressB: {}, - cAddressC: {}, - cAddressD: {}, + addressA: {}, + addressB: {}, + addressC: {}, + addressD: {}, }, } @@ -95,8 +95,8 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { t.Run("address invalidator A invalidates all but D", func(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ - FrozenAccounts: []common.Address{ - cAddressA, + FrozenAccounts: []flow.Address{ + addressA, }, }.ProgramInvalidator() @@ -109,8 +109,8 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { t.Run("address invalidator D invalidates D, C", func(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ - FrozenAccounts: []common.Address{ - cAddressD, + FrozenAccounts: []flow.Address{ + addressD, }, }.ProgramInvalidator() @@ -123,8 +123,8 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { t.Run("address invalidator B invalidates B, C", func(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ - FrozenAccounts: []common.Address{ - cAddressB, + FrozenAccounts: []flow.Address{ + addressB, }, }.ProgramInvalidator() @@ -139,7 +139,7 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ ContractUpdateKeys: []environment.ContractUpdateKey{ { - Address: cAddressA, + Address: addressA, Name: "A", }, }, @@ -156,7 +156,7 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ ContractUpdateKeys: []environment.ContractUpdateKey{ { - Address: cAddressC, + Address: addressC, Name: "C", }, }, @@ -173,7 +173,7 @@ func TestDerivedDataProgramInvalidator(t *testing.T) { invalidator := environment.DerivedDataInvalidator{ ContractUpdateKeys: []environment.ContractUpdateKey{ { - Address: cAddressD, + Address: addressD, Name: "D", }, }, @@ -264,11 +264,11 @@ func TestMeterParamOverridesUpdated(t *testing.T) { ctx.TxBody = &flow.TransactionBody{} - checkForUpdates := func(owner string, key string, expected bool) { + checkForUpdates := func(id flow.RegisterID, expected bool) { view := utils.NewSimpleView() txnState := state.NewTransactionState(view, state.DefaultParameters()) - err := txnState.Set(owner, key, flow.RegisterValue("blah"), false) + err := txnState.Set(id, flow.RegisterValue("blah")) require.NoError(t, err) env := environment.NewTransactionEnvironment( @@ -282,24 +282,18 @@ func TestMeterParamOverridesUpdated(t *testing.T) { } for registerId := range view.Ledger.RegisterTouches { - checkForUpdates(registerId.Owner, registerId.Key, true) - checkForUpdates("other owner", registerId.Key, false) + checkForUpdates(registerId, true) + checkForUpdates( + flow.NewRegisterID("other owner", registerId.Key), + false) } - stabIndex := "$12345678" - require.True(t, state.IsSlabIndex(stabIndex)) + owner := string(ctx.Chain.ServiceAddress().Bytes()) + stabIndexKey := flow.NewRegisterID(owner, "$12345678") + require.True(t, stabIndexKey.IsSlabIndex()) - checkForUpdates( - string(ctx.Chain.ServiceAddress().Bytes()), - stabIndex, - true) - - checkForUpdates( - string(ctx.Chain.ServiceAddress().Bytes()), - "other keys", - false) - - checkForUpdates("other owner", stabIndex, false) - - checkForUpdates("other owner", "other key", false) + checkForUpdates(stabIndexKey, true) + checkForUpdates(flow.NewRegisterID(owner, "other keys"), false) + checkForUpdates(flow.NewRegisterID("other owner", stabIndexKey.Key), false) + checkForUpdates(flow.NewRegisterID("other owner", "other key"), false) } diff --git a/fvm/environment/env.go b/fvm/environment/env.go index bda83f20c5a..43d4ed32996 100644 --- a/fvm/environment/env.go +++ b/fvm/environment/env.go @@ -38,8 +38,9 @@ type Environment interface { Logs() []string // EventEmitter - Events() []flow.Event - ServiceEvents() []flow.Event + Events() flow.EventsList + ServiceEvents() flow.EventsList + ConvertedServiceEvents() flow.ServiceEventList // SystemContracts AccountsStorageCapacity( diff --git a/fvm/environment/event_emitter.go b/fvm/environment/event_emitter.go index 0d02eb0ace7..d1a48a719dc 100644 --- a/fvm/environment/event_emitter.go +++ b/fvm/environment/event_emitter.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" ) @@ -41,8 +42,9 @@ type EventEmitter interface { // OperationNotSupportedError. EmitEvent(event cadence.Event) error - Events() []flow.Event - ServiceEvents() []flow.Event + Events() flow.EventsList + ServiceEvents() flow.EventsList + ConvertedServiceEvents() flow.ServiceEventList Reset() } @@ -70,14 +72,18 @@ func (emitter ParseRestrictedEventEmitter) EmitEvent(event cadence.Event) error event) } -func (emitter ParseRestrictedEventEmitter) Events() []flow.Event { +func (emitter ParseRestrictedEventEmitter) Events() flow.EventsList { return emitter.impl.Events() } -func (emitter ParseRestrictedEventEmitter) ServiceEvents() []flow.Event { +func (emitter ParseRestrictedEventEmitter) ServiceEvents() flow.EventsList { return emitter.impl.ServiceEvents() } +func (emitter ParseRestrictedEventEmitter) ConvertedServiceEvents() flow.ServiceEventList { + return emitter.impl.ConvertedServiceEvents() +} + func (emitter ParseRestrictedEventEmitter) Reset() { emitter.impl.Reset() } @@ -92,12 +98,16 @@ func (NoEventEmitter) EmitEvent(event cadence.Event) error { return nil } -func (NoEventEmitter) Events() []flow.Event { - return []flow.Event{} +func (NoEventEmitter) Events() flow.EventsList { + return flow.EventsList{} +} + +func (NoEventEmitter) ServiceEvents() flow.EventsList { + return flow.EventsList{} } -func (NoEventEmitter) ServiceEvents() []flow.Event { - return []flow.Event{} +func (NoEventEmitter) ConvertedServiceEvents() flow.ServiceEventList { + return flow.ServiceEventList{} } func (NoEventEmitter) Reset() { @@ -181,7 +191,11 @@ func (emitter *eventEmitter) EmitEvent(event cadence.Event) error { return fmt.Errorf("unable to check service event: %w", err) } if ok { - eventEmitError := emitter.eventCollection.AppendServiceEvent(flowEvent, payloadSize) + eventEmitError := emitter.eventCollection.AppendServiceEvent( + emitter.chain, + flowEvent, + payloadSize) + // skip limit if payer is service account if !isServiceAccount && eventEmitError != nil { return eventEmitError @@ -200,31 +214,37 @@ func (emitter *eventEmitter) EmitEvent(event cadence.Event) error { return nil } -func (emitter *eventEmitter) Events() []flow.Event { +func (emitter *eventEmitter) Events() flow.EventsList { return emitter.eventCollection.events } -func (emitter *eventEmitter) ServiceEvents() []flow.Event { +func (emitter *eventEmitter) ServiceEvents() flow.EventsList { return emitter.eventCollection.serviceEvents } +func (emitter *eventEmitter) ConvertedServiceEvents() flow.ServiceEventList { + return emitter.eventCollection.convertedServiceEvents +} + type EventCollection struct { - events flow.EventsList - serviceEvents flow.EventsList - eventCounter uint32 - meter Meter + events flow.EventsList + serviceEvents flow.EventsList + convertedServiceEvents flow.ServiceEventList + eventCounter uint32 + meter Meter } func NewEventCollection(meter Meter) *EventCollection { return &EventCollection{ - events: make([]flow.Event, 0, 10), - serviceEvents: make([]flow.Event, 0, 10), - eventCounter: uint32(0), - meter: meter, + events: make(flow.EventsList, 0, 10), + serviceEvents: make(flow.EventsList, 0, 10), + convertedServiceEvents: make(flow.ServiceEventList, 0, 10), + eventCounter: uint32(0), + meter: meter, } } -func (collection *EventCollection) Events() []flow.Event { +func (collection *EventCollection) Events() flow.EventsList { return collection.events } @@ -234,15 +254,28 @@ func (collection *EventCollection) AppendEvent(event flow.Event, size uint64) er return collection.meter.MeterEmittedEvent(size) } -func (collection *EventCollection) ServiceEvents() []flow.Event { +func (collection *EventCollection) ServiceEvents() flow.EventsList { return collection.serviceEvents } +func (collection *EventCollection) ConvertedServiceEvents() flow.ServiceEventList { + return collection.convertedServiceEvents +} + func (collection *EventCollection) AppendServiceEvent( + chain flow.Chain, event flow.Event, size uint64, ) error { + convertedEvent, err := convert.ServiceEvent(chain.ChainID(), event) + if err != nil { + return fmt.Errorf("could not convert service event: %w", err) + } + collection.serviceEvents = append(collection.serviceEvents, event) + collection.convertedServiceEvents = append( + collection.convertedServiceEvents, + *convertedEvent) collection.eventCounter++ return collection.meter.MeterEmittedEvent(size) } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 147062aea7a..a27f176dada 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -128,6 +128,7 @@ func newFacadeEnvironment( Programs: NewPrograms( tracer, meter, + params.MetricsReporter, txnState, accounts, derivedTxnData), @@ -203,7 +204,7 @@ func NewTransactionEnvironment( tracer, env.Meter, env.accounts, - env.TransactionInfo, + params.TransactionInfoParams.TxBody.Authorizers, params.Chain, params.ContractUpdaterParams, env.ProgramLogger, diff --git a/fvm/environment/meter.go b/fvm/environment/meter.go index fd1d0bc1193..78502bd1e14 100644 --- a/fvm/environment/meter.go +++ b/fvm/environment/meter.go @@ -73,10 +73,7 @@ func (meter *meterImpl) MeterComputation( kind common.ComputationKind, intensity uint, ) error { - if meter.txnState.EnforceLimits() { - return meter.txnState.MeterComputation(kind, intensity) - } - return nil + return meter.txnState.MeterComputation(kind, intensity) } func (meter *meterImpl) ComputationIntensities() meter.MeteredComputationIntensities { @@ -88,10 +85,7 @@ func (meter *meterImpl) ComputationUsed() uint64 { } func (meter *meterImpl) MeterMemory(usage common.MemoryUsage) error { - if meter.txnState.EnforceLimits() { - return meter.txnState.MeterMemory(usage.Kind, uint(usage.Amount)) - } - return nil + return meter.txnState.MeterMemory(usage.Kind, uint(usage.Amount)) } func (meter *meterImpl) MemoryEstimate() uint64 { @@ -99,10 +93,7 @@ func (meter *meterImpl) MemoryEstimate() uint64 { } func (meter *meterImpl) MeterEmittedEvent(byteSize uint64) error { - if meter.txnState.EnforceLimits() { - return meter.txnState.MeterEmittedEvent(byteSize) - } - return nil + return meter.txnState.MeterEmittedEvent(byteSize) } func (meter *meterImpl) TotalEmittedEventBytes() uint64 { diff --git a/fvm/environment/mock/account_freezer.go b/fvm/environment/mock/account_freezer.go index a2dfd971671..089a2295f91 100644 --- a/fvm/environment/mock/account_freezer.go +++ b/fvm/environment/mock/account_freezer.go @@ -5,6 +5,8 @@ package mock import ( common "github.com/onflow/cadence/runtime/common" + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" ) @@ -14,15 +16,15 @@ type AccountFreezer struct { } // FrozenAccounts provides a mock function with given fields: -func (_m *AccountFreezer) FrozenAccounts() []common.Address { +func (_m *AccountFreezer) FrozenAccounts() []flow.Address { ret := _m.Called() - var r0 []common.Address - if rf, ok := ret.Get(0).(func() []common.Address); ok { + var r0 []flow.Address + if rf, ok := ret.Get(0).(func() []flow.Address); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]common.Address) + r0 = ret.Get(0).([]flow.Address) } } diff --git a/fvm/environment/mock/account_key_updater.go b/fvm/environment/mock/account_key_updater.go index 93b02f58586..6eb43705c3c 100644 --- a/fvm/environment/mock/account_key_updater.go +++ b/fvm/environment/mock/account_key_updater.go @@ -17,13 +17,13 @@ type AccountKeyUpdater struct { mock.Mock } -// AddAccountKey provides a mock function with given fields: address, publicKey, hashAlgo, weight -func (_m *AccountKeyUpdater) AddAccountKey(address common.Address, publicKey *stdlib.PublicKey, hashAlgo sema.HashAlgorithm, weight int) (*stdlib.AccountKey, error) { - ret := _m.Called(address, publicKey, hashAlgo, weight) +// AddAccountKey provides a mock function with given fields: runtimeAddress, publicKey, hashAlgo, weight +func (_m *AccountKeyUpdater) AddAccountKey(runtimeAddress common.Address, publicKey *stdlib.PublicKey, hashAlgo sema.HashAlgorithm, weight int) (*stdlib.AccountKey, error) { + ret := _m.Called(runtimeAddress, publicKey, hashAlgo, weight) var r0 *stdlib.AccountKey if rf, ok := ret.Get(0).(func(common.Address, *stdlib.PublicKey, sema.HashAlgorithm, int) *stdlib.AccountKey); ok { - r0 = rf(address, publicKey, hashAlgo, weight) + r0 = rf(runtimeAddress, publicKey, hashAlgo, weight) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*stdlib.AccountKey) @@ -32,7 +32,7 @@ func (_m *AccountKeyUpdater) AddAccountKey(address common.Address, publicKey *st var r1 error if rf, ok := ret.Get(1).(func(common.Address, *stdlib.PublicKey, sema.HashAlgorithm, int) error); ok { - r1 = rf(address, publicKey, hashAlgo, weight) + r1 = rf(runtimeAddress, publicKey, hashAlgo, weight) } else { r1 = ret.Error(1) } @@ -40,13 +40,13 @@ func (_m *AccountKeyUpdater) AddAccountKey(address common.Address, publicKey *st return r0, r1 } -// AddEncodedAccountKey provides a mock function with given fields: address, publicKey -func (_m *AccountKeyUpdater) AddEncodedAccountKey(address common.Address, publicKey []byte) error { - ret := _m.Called(address, publicKey) +// AddEncodedAccountKey provides a mock function with given fields: runtimeAddress, publicKey +func (_m *AccountKeyUpdater) AddEncodedAccountKey(runtimeAddress common.Address, publicKey []byte) error { + ret := _m.Called(runtimeAddress, publicKey) var r0 error if rf, ok := ret.Get(0).(func(common.Address, []byte) error); ok { - r0 = rf(address, publicKey) + r0 = rf(runtimeAddress, publicKey) } else { r0 = ret.Error(0) } @@ -54,13 +54,13 @@ func (_m *AccountKeyUpdater) AddEncodedAccountKey(address common.Address, public return r0 } -// RevokeAccountKey provides a mock function with given fields: address, keyIndex -func (_m *AccountKeyUpdater) RevokeAccountKey(address common.Address, keyIndex int) (*stdlib.AccountKey, error) { - ret := _m.Called(address, keyIndex) +// RevokeAccountKey provides a mock function with given fields: runtimeAddress, keyIndex +func (_m *AccountKeyUpdater) RevokeAccountKey(runtimeAddress common.Address, keyIndex int) (*stdlib.AccountKey, error) { + ret := _m.Called(runtimeAddress, keyIndex) var r0 *stdlib.AccountKey if rf, ok := ret.Get(0).(func(common.Address, int) *stdlib.AccountKey); ok { - r0 = rf(address, keyIndex) + r0 = rf(runtimeAddress, keyIndex) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*stdlib.AccountKey) @@ -69,7 +69,7 @@ func (_m *AccountKeyUpdater) RevokeAccountKey(address common.Address, keyIndex i var r1 error if rf, ok := ret.Get(1).(func(common.Address, int) error); ok { - r1 = rf(address, keyIndex) + r1 = rf(runtimeAddress, keyIndex) } else { r1 = ret.Error(1) } @@ -77,13 +77,13 @@ func (_m *AccountKeyUpdater) RevokeAccountKey(address common.Address, keyIndex i return r0, r1 } -// RevokeEncodedAccountKey provides a mock function with given fields: address, index -func (_m *AccountKeyUpdater) RevokeEncodedAccountKey(address common.Address, index int) ([]byte, error) { - ret := _m.Called(address, index) +// RevokeEncodedAccountKey provides a mock function with given fields: runtimeAddress, index +func (_m *AccountKeyUpdater) RevokeEncodedAccountKey(runtimeAddress common.Address, index int) ([]byte, error) { + ret := _m.Called(runtimeAddress, index) var r0 []byte if rf, ok := ret.Get(0).(func(common.Address, int) []byte); ok { - r0 = rf(address, index) + r0 = rf(runtimeAddress, index) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) @@ -92,7 +92,7 @@ func (_m *AccountKeyUpdater) RevokeEncodedAccountKey(address common.Address, ind var r1 error if rf, ok := ret.Get(1).(func(common.Address, int) error); ok { - r1 = rf(address, index) + r1 = rf(runtimeAddress, index) } else { r1 = ret.Error(1) } diff --git a/fvm/environment/mock/accounts.go b/fvm/environment/mock/accounts.go index cc780d263f5..c5638296fef 100644 --- a/fvm/environment/mock/accounts.go +++ b/fvm/environment/mock/accounts.go @@ -268,13 +268,13 @@ func (_m *Accounts) GetStorageUsed(address flow.Address) (uint64, error) { return r0, r1 } -// GetValue provides a mock function with given fields: address, key -func (_m *Accounts) GetValue(address flow.Address, key string) ([]byte, error) { - ret := _m.Called(address, key) +// GetValue provides a mock function with given fields: id +func (_m *Accounts) GetValue(id flow.RegisterID) ([]byte, error) { + ret := _m.Called(id) var r0 []byte - if rf, ok := ret.Get(0).(func(flow.Address, string) []byte); ok { - r0 = rf(address, key) + if rf, ok := ret.Get(0).(func(flow.RegisterID) []byte); ok { + r0 = rf(id) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]byte) @@ -282,8 +282,8 @@ func (_m *Accounts) GetValue(address flow.Address, key string) ([]byte, error) { } var r1 error - if rf, ok := ret.Get(1).(func(flow.Address, string) error); ok { - r1 = rf(address, key) + if rf, ok := ret.Get(1).(func(flow.RegisterID) error); ok { + r1 = rf(id) } else { r1 = ret.Error(1) } @@ -342,13 +342,13 @@ func (_m *Accounts) SetPublicKey(address flow.Address, keyIndex uint64, publicKe return r0, r1 } -// SetValue provides a mock function with given fields: address, key, value -func (_m *Accounts) SetValue(address flow.Address, key string, value []byte) error { - ret := _m.Called(address, key, value) +// SetValue provides a mock function with given fields: id, value +func (_m *Accounts) SetValue(id flow.RegisterID, value []byte) error { + ret := _m.Called(id, value) var r0 error - if rf, ok := ret.Get(0).(func(flow.Address, string, []byte) error); ok { - r0 = rf(address, key, value) + if rf, ok := ret.Get(0).(func(flow.RegisterID, []byte) error); ok { + r0 = rf(id, value) } else { r0 = ret.Error(0) } diff --git a/fvm/environment/mock/contract_updater_stubs.go b/fvm/environment/mock/contract_updater_stubs.go index 91e5f166b87..d5969eaafcf 100644 --- a/fvm/environment/mock/contract_updater_stubs.go +++ b/fvm/environment/mock/contract_updater_stubs.go @@ -4,7 +4,8 @@ package mock import ( cadence "github.com/onflow/cadence" - common "github.com/onflow/cadence/runtime/common" + + flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" ) @@ -15,15 +16,15 @@ type ContractUpdaterStubs struct { } // GetAuthorizedAccounts provides a mock function with given fields: path -func (_m *ContractUpdaterStubs) GetAuthorizedAccounts(path cadence.Path) []common.Address { +func (_m *ContractUpdaterStubs) GetAuthorizedAccounts(path cadence.Path) []flow.Address { ret := _m.Called(path) - var r0 []common.Address - if rf, ok := ret.Get(0).(func(cadence.Path) []common.Address); ok { + var r0 []flow.Address + if rf, ok := ret.Get(0).(func(cadence.Path) []flow.Address); ok { r0 = rf(path) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]common.Address) + r0 = ret.Get(0).([]flow.Address) } } @@ -59,18 +60,18 @@ func (_m *ContractUpdaterStubs) RestrictedRemovalEnabled() bool { } // UseContractAuditVoucher provides a mock function with given fields: address, code -func (_m *ContractUpdaterStubs) UseContractAuditVoucher(address common.Address, code []byte) (bool, error) { +func (_m *ContractUpdaterStubs) UseContractAuditVoucher(address flow.Address, code []byte) (bool, error) { ret := _m.Called(address, code) var r0 bool - if rf, ok := ret.Get(0).(func(common.Address, []byte) bool); ok { + if rf, ok := ret.Get(0).(func(flow.Address, []byte) bool); ok { r0 = rf(address, code) } else { r0 = ret.Get(0).(bool) } var r1 error - if rf, ok := ret.Get(1).(func(common.Address, []byte) error); ok { + if rf, ok := ret.Get(1).(func(flow.Address, []byte) error); ok { r1 = rf(address, code) } else { r1 = ret.Error(1) diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 2881189661f..a7e2f339c31 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -284,6 +284,22 @@ func (_m *Environment) ComputationUsed() uint64 { return r0 } +// ConvertedServiceEvents provides a mock function with given fields: +func (_m *Environment) ConvertedServiceEvents() flow.ServiceEventList { + ret := _m.Called() + + var r0 flow.ServiceEventList + if rf, ok := ret.Get(0).(func() flow.ServiceEventList); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.ServiceEventList) + } + } + + return r0 +} + // CreateAccount provides a mock function with given fields: payer func (_m *Environment) CreateAccount(payer common.Address) (common.Address, error) { ret := _m.Called(payer) @@ -368,15 +384,15 @@ func (_m *Environment) EmitEvent(_a0 cadence.Event) error { } // Events provides a mock function with given fields: -func (_m *Environment) Events() []flow.Event { +func (_m *Environment) Events() flow.EventsList { ret := _m.Called() - var r0 []flow.Event - if rf, ok := ret.Get(0).(func() []flow.Event); ok { + var r0 flow.EventsList + if rf, ok := ret.Get(0).(func() flow.EventsList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.Event) + r0 = ret.Get(0).(flow.EventsList) } } @@ -407,15 +423,15 @@ func (_m *Environment) FlushPendingUpdates() (derived.TransactionInvalidator, er } // FrozenAccounts provides a mock function with given fields: -func (_m *Environment) FrozenAccounts() []common.Address { +func (_m *Environment) FrozenAccounts() []flow.Address { ret := _m.Called() - var r0 []common.Address - if rf, ok := ret.Get(0).(func() []common.Address); ok { + var r0 []flow.Address + if rf, ok := ret.Get(0).(func() []flow.Address); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]common.Address) + r0 = ret.Get(0).([]flow.Address) } } @@ -1047,15 +1063,15 @@ func (_m *Environment) RevokeEncodedAccountKey(address common.Address, index int } // ServiceEvents provides a mock function with given fields: -func (_m *Environment) ServiceEvents() []flow.Event { +func (_m *Environment) ServiceEvents() flow.EventsList { ret := _m.Called() - var r0 []flow.Event - if rf, ok := ret.Get(0).(func() []flow.Event); ok { + var r0 flow.EventsList + if rf, ok := ret.Get(0).(func() flow.EventsList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.Event) + r0 = ret.Get(0).(flow.EventsList) } } @@ -1109,22 +1125,6 @@ func (_m *Environment) SetValue(owner []byte, key []byte, value []byte) error { return r0 } -// SigningAccounts provides a mock function with given fields: -func (_m *Environment) SigningAccounts() []common.Address { - ret := _m.Called() - - var r0 []common.Address - if rf, ok := ret.Get(0).(func() []common.Address); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]common.Address) - } - } - - return r0 -} - // StartChildSpan provides a mock function with given fields: name, options func (_m *Environment) StartChildSpan(name trace.SpanName, options ...oteltrace.SpanStartOption) tracing.TracerSpan { _va := make([]interface{}, len(options)) diff --git a/fvm/environment/mock/event_emitter.go b/fvm/environment/mock/event_emitter.go index fabfc2988a4..fdaba4521b5 100644 --- a/fvm/environment/mock/event_emitter.go +++ b/fvm/environment/mock/event_emitter.go @@ -15,6 +15,22 @@ type EventEmitter struct { mock.Mock } +// ConvertedServiceEvents provides a mock function with given fields: +func (_m *EventEmitter) ConvertedServiceEvents() flow.ServiceEventList { + ret := _m.Called() + + var r0 flow.ServiceEventList + if rf, ok := ret.Get(0).(func() flow.ServiceEventList); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.ServiceEventList) + } + } + + return r0 +} + // EmitEvent provides a mock function with given fields: event func (_m *EventEmitter) EmitEvent(event cadence.Event) error { ret := _m.Called(event) @@ -30,15 +46,15 @@ func (_m *EventEmitter) EmitEvent(event cadence.Event) error { } // Events provides a mock function with given fields: -func (_m *EventEmitter) Events() []flow.Event { +func (_m *EventEmitter) Events() flow.EventsList { ret := _m.Called() - var r0 []flow.Event - if rf, ok := ret.Get(0).(func() []flow.Event); ok { + var r0 flow.EventsList + if rf, ok := ret.Get(0).(func() flow.EventsList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.Event) + r0 = ret.Get(0).(flow.EventsList) } } @@ -51,15 +67,15 @@ func (_m *EventEmitter) Reset() { } // ServiceEvents provides a mock function with given fields: -func (_m *EventEmitter) ServiceEvents() []flow.Event { +func (_m *EventEmitter) ServiceEvents() flow.EventsList { ret := _m.Called() - var r0 []flow.Event - if rf, ok := ret.Get(0).(func() []flow.Event); ok { + var r0 flow.EventsList + if rf, ok := ret.Get(0).(func() flow.EventsList); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.Event) + r0 = ret.Get(0).(flow.EventsList) } } diff --git a/fvm/environment/mock/metrics_reporter.go b/fvm/environment/mock/metrics_reporter.go index 0b899927049..bacd2d86ad0 100644 --- a/fvm/environment/mock/metrics_reporter.go +++ b/fvm/environment/mock/metrics_reporter.go @@ -33,6 +33,16 @@ func (_m *MetricsReporter) RuntimeTransactionParsed(_a0 time.Duration) { _m.Called(_a0) } +// RuntimeTransactionProgramsCacheHit provides a mock function with given fields: +func (_m *MetricsReporter) RuntimeTransactionProgramsCacheHit() { + _m.Called() +} + +// RuntimeTransactionProgramsCacheMiss provides a mock function with given fields: +func (_m *MetricsReporter) RuntimeTransactionProgramsCacheMiss() { + _m.Called() +} + type mockConstructorTestingTNewMetricsReporter interface { mock.TestingT Cleanup(func()) diff --git a/fvm/environment/mock/transaction_info.go b/fvm/environment/mock/transaction_info.go index 1d8a15069aa..4db8437c9b9 100644 --- a/fvm/environment/mock/transaction_info.go +++ b/fvm/environment/mock/transaction_info.go @@ -66,22 +66,6 @@ func (_m *TransactionInfo) LimitAccountStorage() bool { return r0 } -// SigningAccounts provides a mock function with given fields: -func (_m *TransactionInfo) SigningAccounts() []common.Address { - ret := _m.Called() - - var r0 []common.Address - if rf, ok := ret.Get(0).(func() []common.Address); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]common.Address) - } - } - - return r0 -} - // TransactionFeesEnabled provides a mock function with given fields: func (_m *TransactionInfo) TransactionFeesEnabled() bool { ret := _m.Called() diff --git a/fvm/environment/program_logger.go b/fvm/environment/program_logger.go index 6783a910b47..44b7e859bcf 100644 --- a/fvm/environment/program_logger.go +++ b/fvm/environment/program_logger.go @@ -19,6 +19,8 @@ type MetricsReporter interface { RuntimeTransactionChecked(time.Duration) RuntimeTransactionInterpreted(time.Duration) RuntimeSetNumberOfAccounts(count uint64) + RuntimeTransactionProgramsCacheMiss() + RuntimeTransactionProgramsCacheHit() } // NoopMetricsReporter is a MetricReporter that does nothing. @@ -36,6 +38,12 @@ func (NoopMetricsReporter) RuntimeTransactionInterpreted(time.Duration) {} // RuntimeSetNumberOfAccounts is a noop func (NoopMetricsReporter) RuntimeSetNumberOfAccounts(count uint64) {} +// RuntimeTransactionProgramsCacheMiss is a noop +func (NoopMetricsReporter) RuntimeTransactionProgramsCacheMiss() {} + +// RuntimeTransactionProgramsCacheHit is a noop +func (NoopMetricsReporter) RuntimeTransactionProgramsCacheHit() {} + type ProgramLoggerParams struct { zerolog.Logger @@ -85,6 +93,15 @@ func (logger *ProgramLogger) ProgramLog(message string) error { trace.FVMEnvProgramLog).End() if logger.CadenceLoggingEnabled { + + // If cadence logging is enabled (which is usually in the + // emulator or emulator based tools), + // we log the message to the zerolog logger so that they can be tracked + // while stepping through a transaction/script. + logger.Logger(). + Debug(). + Msgf("Cadence log: %s", message) + logger.logs = append(logger.logs, message) } return nil diff --git a/fvm/environment/programs.go b/fvm/environment/programs.go index 07b61330323..2460cb6494e 100644 --- a/fvm/environment/programs.go +++ b/fvm/environment/programs.go @@ -30,8 +30,9 @@ type DerivedTransactionData interface { // these nested transactions on Set calls in order to capture the states // needed for parsing the programs. type Programs struct { - tracer tracing.TracerSpan - meter Meter + tracer tracing.TracerSpan + meter Meter + metrics MetricsReporter txnState *state.TransactionState accounts Accounts @@ -50,6 +51,7 @@ type Programs struct { func NewPrograms( tracer tracing.TracerSpan, meter Meter, + metrics MetricsReporter, txnState *state.TransactionState, accounts Accounts, derivedTxnData DerivedTransactionData, @@ -57,6 +59,7 @@ func NewPrograms( return &Programs{ tracer: tracer, meter: meter, + metrics: metrics, txnState: txnState, accounts: accounts, derivedTxnData: derivedTxnData, @@ -88,6 +91,12 @@ func (programs *Programs) set( return err } + if state.BytesWritten() > 0 { + // This should never happen. Loading a program should not write to the state. + // If this happens, it indicates an implementation error. + return fmt.Errorf("cannot set program. State was written to during program parsing") + } + // Get collected dependencies of the loaded program. stackLocation, dependencies, err := programs.dependencyStack.pop() if err != nil { @@ -134,6 +143,8 @@ func (programs *Programs) get( program, state, has := programs.derivedTxnData.GetProgram(address) if has { + programs.cacheHit() + programs.dependencyStack.addDependencies(program.Dependencies) err := programs.txnState.AttachAndCommit(state) if err != nil { @@ -144,6 +155,7 @@ func (programs *Programs) get( return program.Program, true } + programs.cacheMiss() // this program is not in cache, so we need to load it into the cache. // tho have proper invalidation, we need to track the dependencies of the program. @@ -178,7 +190,7 @@ func (programs *Programs) GetProgram( } if addressLocation, ok := location.(common.AddressLocation); ok { - address := flow.Address(addressLocation.Address) + address := flow.ConvertAddress(addressLocation.Address) freezeError := programs.accounts.CheckAccountNotFrozen(address) if freezeError != nil { @@ -234,6 +246,14 @@ func (programs *Programs) DecodeArgument( return v, err } +func (programs *Programs) cacheHit() { + programs.metrics.RuntimeTransactionProgramsCacheHit() +} + +func (programs *Programs) cacheMiss() { + programs.metrics.RuntimeTransactionProgramsCacheMiss() +} + // dependencyTracker tracks dependencies for a location // Or in other words it builds up a list of dependencies for the program being loaded. // If a program imports another program (A imports B), then B is a dependency of A. @@ -268,7 +288,7 @@ func (s *dependencyStack) push(loc common.AddressLocation) { dependencies := make(derived.ProgramDependencies, 1) // A program is listed as its own dependency. - dependencies.AddDependency(loc.Address) + dependencies.AddDependency(flow.ConvertAddress(loc.Address)) s.trackers = append(s.trackers, dependencyTracker{ location: loc, diff --git a/fvm/environment/programs_test.go b/fvm/environment/programs_test.go index 6057a66dc60..2d1a3148b62 100644 --- a/fvm/environment/programs_test.go +++ b/fvm/environment/programs_test.go @@ -107,9 +107,7 @@ func Test_Programs(t *testing.T) { ).AddAuthorizer(address) } - mainView := delta.NewView(func(_, _ string) (flow.RegisterValue, error) { - return nil, nil - }) + mainView := delta.NewDeltaView(nil) txnState := state.NewTransactionState(mainView, state.DefaultParameters()) @@ -192,13 +190,17 @@ func Test_Programs(t *testing.T) { derivedBlockData.NextTxIndexForTestingOnly()) loadedCode := false - viewExecA := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - if key == environment.ContractKey("A") { - loadedCode = true - } - - return mainView.Peek(owner, key) - }) + viewExecA := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + expectedId := flow.ContractRegisterID( + flow.BytesToAddress([]byte(id.Owner)), + "A") + if id == expectedId { + loadedCode = true + } + + return mainView.Peek(id) + })) err = vm.Run(context, procCallA, viewExecA) require.NoError(t, err) @@ -211,10 +213,12 @@ func Test_Programs(t *testing.T) { entry := derivedBlockData.GetProgramForTestingOnly(contractALocation) require.NotNil(t, entry) + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 1, cached) // assert dependencies are correct require.Len(t, entry.Value.Dependencies, 1) - require.NotNil(t, entry.Value.Dependencies[common.MustBytesToAddress(addressA.Bytes())]) + require.NotNil(t, entry.Value.Dependencies[addressA]) // type assertion for further inspections require.IsType(t, entry.State.View(), &delta.View{}) @@ -231,12 +235,16 @@ func Test_Programs(t *testing.T) { require.NoError(t, err) // execute transaction again, this time make sure it doesn't load code - viewExecA2 := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - // this time we fail if a read of code occurs - require.NotEqual(t, key, environment.ContractKey("A")) + viewExecA2 := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + notId := flow.ContractRegisterID( + flow.BytesToAddress([]byte(id.Owner)), + "A") + // this time we fail if a read of code occurs + require.NotEqual(t, id, notId) - return mainView.Peek(owner, key) - }) + return mainView.Peek(id) + })) procCallA = fvm.Transaction( callTx("A", addressA), @@ -257,7 +265,6 @@ func Test_Programs(t *testing.T) { }) t.Run("deploying another contract invalidates dependant programs", func(t *testing.T) { - // deploy contract B procContractB := fvm.Transaction( contractDeployTx("B", contractBCode, addressB), @@ -274,6 +281,9 @@ func Test_Programs(t *testing.T) { require.Nil(t, entryB) require.Nil(t, entryC) require.NotNil(t, entryA) + + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 1, cached) }) var viewExecB *delta.View @@ -287,7 +297,8 @@ func Test_Programs(t *testing.T) { callTx("B", addressB), derivedBlockData.NextTxIndexForTestingOnly()) - viewExecB = delta.NewView(mainView.Peek) + viewExecB = delta.NewDeltaView( + delta.NewPeekerStorageSnapshot(mainView)) err = vm.Run(context, procCallB, viewExecB) require.NoError(t, err) @@ -308,8 +319,8 @@ func Test_Programs(t *testing.T) { // assert dependencies are correct require.Len(t, entryB.Value.Dependencies, 2) - require.NotNil(t, entryB.Value.Dependencies[common.MustBytesToAddress(addressA.Bytes())]) - require.NotNil(t, entryB.Value.Dependencies[common.MustBytesToAddress(addressB.Bytes())]) + require.NotNil(t, entryB.Value.Dependencies[addressA]) + require.NotNil(t, entryB.Value.Dependencies[addressB]) // program B should contain all the registers used by program A, as it depends on it require.IsType(t, entryB.State.View(), &delta.View{}) @@ -317,7 +328,7 @@ func Test_Programs(t *testing.T) { entriesA := deltaA.Delta().UpdatedRegisters() for _, entry := range entriesA { - v, has := deltaB.Delta().Get(entry.Key.Owner, entry.Key.Key) + v, has := deltaB.Delta().Get(entry.Key) require.True(t, has) require.Equal(t, entry.Value, v) @@ -340,13 +351,20 @@ func Test_Programs(t *testing.T) { // rerun transaction // execute transaction again, this time make sure it doesn't load code - viewExecB2 := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - // this time we fail if a read of code occurs - require.NotEqual(t, key, environment.ContractKey("A")) - require.NotEqual(t, key, environment.ContractKey("B")) - - return mainView.Peek(owner, key) - }) + viewExecB2 := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + idA := flow.ContractRegisterID( + flow.BytesToAddress([]byte(id.Owner)), + "A") + idB := flow.ContractRegisterID( + flow.BytesToAddress([]byte(id.Owner)), + "B") + // this time we fail if a read of code occurs + require.NotEqual(t, id.Key, idA.Key) + require.NotEqual(t, id.Key, idB.Key) + + return mainView.Peek(id) + })) procCallB = fvm.Transaction( callTx("B", addressB), @@ -369,10 +387,14 @@ func Test_Programs(t *testing.T) { // at this point programs cache should contain data for contract A // only because contract B has been called - viewExecA := delta.NewView(func(owner, key string) (flow.RegisterValue, error) { - require.NotEqual(t, key, environment.ContractKey("A")) - return mainView.Peek(owner, key) - }) + viewExecA := delta.NewDeltaView(delta.NewReadFuncStorageSnapshot( + func(id flow.RegisterID) (flow.RegisterValue, error) { + notId := flow.ContractRegisterID( + flow.BytesToAddress([]byte(id.Owner)), + "A") + require.NotEqual(t, id, notId) + return mainView.Peek(id) + })) // run a TX using contract A procCallA := fvm.Transaction( @@ -409,6 +431,8 @@ func Test_Programs(t *testing.T) { require.NotNil(t, entryB) require.Nil(t, entryC) + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 2, cached) }) t.Run("importing C should chain-import B and A", func(t *testing.T) { @@ -416,7 +440,8 @@ func Test_Programs(t *testing.T) { callTx("C", addressC), derivedBlockData.NextTxIndexForTestingOnly()) - viewExecC := delta.NewView(mainView.Peek) + viewExecC := delta.NewDeltaView( + delta.NewPeekerStorageSnapshot(mainView)) err = vm.Run(context, procCallC, viewExecC) require.NoError(t, err) @@ -445,9 +470,12 @@ func Test_Programs(t *testing.T) { // assert dependencies are correct require.Len(t, entryC.Value.Dependencies, 3) - require.NotNil(t, entryC.Value.Dependencies[common.MustBytesToAddress(addressA.Bytes())]) - require.NotNil(t, entryC.Value.Dependencies[common.MustBytesToAddress(addressB.Bytes())]) - require.NotNil(t, entryC.Value.Dependencies[common.MustBytesToAddress(addressC.Bytes())]) + require.NotNil(t, entryC.Value.Dependencies[addressA]) + require.NotNil(t, entryC.Value.Dependencies[addressB]) + require.NotNil(t, entryC.Value.Dependencies[addressC]) + + cached := derivedBlockData.CachedPrograms() + require.Equal(t, 3, cached) }) } diff --git a/fvm/environment/system_contracts.go b/fvm/environment/system_contracts.go index 640d6a60f07..4ad42286b56 100644 --- a/fvm/environment/system_contracts.go +++ b/fvm/environment/system_contracts.go @@ -292,7 +292,7 @@ var useContractAuditVoucherSpec = ContractFunctionSpec{ // UseContractAuditVoucher executes the use a contract deployment audit voucher // contract. func (sys *SystemContracts) UseContractAuditVoucher( - address common.Address, + address flow.Address, code string, ) (bool, error) { resultCdc, err := sys.Invoke( diff --git a/fvm/environment/system_contracts_test.go b/fvm/environment/system_contracts_test.go index dcce8e57319..da2ff207b07 100644 --- a/fvm/environment/system_contracts_test.go +++ b/fvm/environment/system_contracts_test.go @@ -77,7 +77,7 @@ func TestSystemContractsInvoke(t *testing.T) { value, err := invoker.Invoke( environment.ContractFunctionSpec{ AddressFromChain: func(_ flow.Chain) flow.Address { - return flow.Address{} + return flow.EmptyAddress }, FunctionName: "functionName", }, diff --git a/fvm/environment/transaction_info.go b/fvm/environment/transaction_info.go index e7c6d3f7054..f6428dc6a93 100644 --- a/fvm/environment/transaction_info.go +++ b/fvm/environment/transaction_info.go @@ -1,7 +1,7 @@ package environment import ( - "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/state" @@ -40,13 +40,11 @@ type TransactionInfo interface { TransactionFeesEnabled() bool LimitAccountStorage() bool - SigningAccounts() []runtime.Address - IsServiceAccountAuthorizer() bool // Cadence's runtime API. Note that the script variant will return // OperationNotSupportedError. - GetSigningAccounts() ([]runtime.Address, error) + GetSigningAccounts() ([]common.Address, error) } type ParseRestrictedTransactionInfo struct { @@ -80,16 +78,12 @@ func (info ParseRestrictedTransactionInfo) LimitAccountStorage() bool { return info.impl.LimitAccountStorage() } -func (info ParseRestrictedTransactionInfo) SigningAccounts() []runtime.Address { - return info.impl.SigningAccounts() -} - func (info ParseRestrictedTransactionInfo) IsServiceAccountAuthorizer() bool { return info.impl.IsServiceAccountAuthorizer() } func (info ParseRestrictedTransactionInfo) GetSigningAccounts() ( - []runtime.Address, + []common.Address, error, ) { return parseRestrict1Ret( @@ -103,7 +97,7 @@ type transactionInfo struct { tracer tracing.TracerSpan - authorizers []runtime.Address + authorizers []common.Address isServiceAccountAuthorizer bool } @@ -115,12 +109,12 @@ func NewTransactionInfo( isServiceAccountAuthorizer := false runtimeAddresses := make( - []runtime.Address, + []common.Address, 0, len(params.TxBody.Authorizers)) for _, auth := range params.TxBody.Authorizers { - runtimeAddresses = append(runtimeAddresses, runtime.Address(auth)) + runtimeAddresses = append(runtimeAddresses, common.Address(auth)) if auth == serviceAccount { isServiceAccountAuthorizer = true } @@ -150,15 +144,11 @@ func (info *transactionInfo) LimitAccountStorage() bool { return info.params.LimitAccountStorage } -func (info *transactionInfo) SigningAccounts() []runtime.Address { - return info.authorizers -} - func (info *transactionInfo) IsServiceAccountAuthorizer() bool { return info.isServiceAccountAuthorizer } -func (info *transactionInfo) GetSigningAccounts() ([]runtime.Address, error) { +func (info *transactionInfo) GetSigningAccounts() ([]common.Address, error) { defer info.tracer.StartExtensiveTracingChildSpan( trace.FVMEnvGetSigningAccounts).End() @@ -187,14 +177,10 @@ func (NoTransactionInfo) LimitAccountStorage() bool { return false } -func (NoTransactionInfo) SigningAccounts() []runtime.Address { - return nil -} - func (NoTransactionInfo) IsServiceAccountAuthorizer() bool { return false } -func (NoTransactionInfo) GetSigningAccounts() ([]runtime.Address, error) { +func (NoTransactionInfo) GetSigningAccounts() ([]common.Address, error) { return nil, errors.NewOperationNotSupportedError("GetSigningAccounts") } diff --git a/fvm/environment/uuids.go b/fvm/environment/uuids.go index cf0b5aeb63d..446ebb654b6 100644 --- a/fvm/environment/uuids.go +++ b/fvm/environment/uuids.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/utils/slices" ) @@ -57,10 +58,7 @@ func NewUUIDGenerator( // GetUUID reads uint64 byte value for uuid from the state func (generator *uUIDGenerator) getUUID() (uint64, error) { - stateBytes, err := generator.txnState.Get( - "", - state.UUIDKey, - generator.txnState.EnforceLimits()) + stateBytes, err := generator.txnState.Get(flow.UUIDRegisterID) if err != nil { return 0, fmt.Errorf("cannot get uuid byte from state: %w", err) } @@ -73,11 +71,7 @@ func (generator *uUIDGenerator) getUUID() (uint64, error) { func (generator *uUIDGenerator) setUUID(uuid uint64) error { bytes := make([]byte, 8) binary.BigEndian.PutUint64(bytes, uuid) - err := generator.txnState.Set( - "", - state.UUIDKey, - bytes, - generator.txnState.EnforceLimits()) + err := generator.txnState.Set(flow.UUIDRegisterID, bytes) if err != nil { return fmt.Errorf("cannot set uuid byte to state: %w", err) } diff --git a/fvm/environment/value_store.go b/fvm/environment/value_store.go index b6f10361ecd..bfd4c20fdf4 100644 --- a/fvm/environment/value_store.go +++ b/fvm/environment/value_store.go @@ -123,14 +123,12 @@ func (store *valueStore) GetValue( ) { defer store.tracer.StartChildSpan(trace.FVMEnvGetValue).End() - key := string(keyBytes) - - address := flow.BytesToAddress(owner) - if state.IsFVMStateKey(string(owner), key) { - return nil, errors.NewInvalidFVMStateAccessError(address, key, "read") + id := flow.CadenceRegisterID(owner, keyBytes) + if id.IsInternalState() { + return nil, errors.NewInvalidInternalStateAccessError(id, "read") } - v, err := store.accounts.GetValue(address, key) + v, err := store.accounts.GetValue(id) if err != nil { return nil, fmt.Errorf("get value failed: %w", err) } @@ -150,11 +148,9 @@ func (store *valueStore) SetValue( ) error { defer store.tracer.StartChildSpan(trace.FVMEnvSetValue).End() - key := string(keyBytes) - - address := flow.BytesToAddress(owner) - if state.IsFVMStateKey(string(owner), key) { - return errors.NewInvalidFVMStateAccessError(address, key, "modify") + id := flow.CadenceRegisterID(owner, keyBytes) + if id.IsInternalState() { + return errors.NewInvalidInternalStateAccessError(id, "modify") } err := store.meter.MeterComputation( @@ -164,7 +160,7 @@ func (store *valueStore) SetValue( return fmt.Errorf("set value failed: %w", err) } - err = store.accounts.SetValue(address, key, value) + err = store.accounts.SetValue(id, value) if err != nil { return fmt.Errorf("set value failed: %w", err) } diff --git a/fvm/errors/codes.go b/fvm/errors/codes.go index 0b9e7dbf178..5714e3ed589 100644 --- a/fvm/errors/codes.go +++ b/fvm/errors/codes.go @@ -79,7 +79,7 @@ const ( ErrCodeScriptExecutionCancelledError ErrorCode = 1114 ErrCodeScriptExecutionTimedOutError ErrorCode = 1113 ErrCodeEventEncodingError ErrorCode = 1115 - ErrCodeInvalidFVMStateAccessError ErrorCode = 1116 + ErrCodeInvalidInternalStateAccessError ErrorCode = 1116 // 1117 was never deployed and is free to use ErrCodeInsufficientPayerBalance ErrorCode = 1118 diff --git a/fvm/errors/execution.go b/fvm/errors/execution.go index 4311e5e72db..4f36d4c5b85 100644 --- a/fvm/errors/execution.go +++ b/fvm/errors/execution.go @@ -1,8 +1,6 @@ package errors import ( - "strings" - "github.com/onflow/cadence" "github.com/onflow/cadence/runtime" @@ -150,15 +148,14 @@ func NewEventLimitExceededError( // NewStateKeySizeLimitError constructs a CodedError which indicates that the // provided key has exceeded the size limit allowed by the storage. func NewStateKeySizeLimitError( - owner string, - key string, + id flow.RegisterID, size uint64, limit uint64, ) CodedError { return NewCodedError( ErrCodeStateKeySizeLimitError, "key %s has size %d which is higher than storage key size limit %d.", - strings.Join([]string{owner, key}, "/"), + id, size, limit) } @@ -257,17 +254,16 @@ func NewCouldNotGetExecutionParameterFromStateError( path) } -// NewInvalidFVMStateAccessError constructs a new CodedError which indicates -// that the cadence program attempted to directly access fvm's internal state. -func NewInvalidFVMStateAccessError( - address flow.Address, - path string, +// NewInvalidInternalStateAccessError constructs a new CodedError which +// indicates that the cadence program attempted to directly access flow's +// internal state. +func NewInvalidInternalStateAccessError( + id flow.RegisterID, opType string, ) CodedError { return NewCodedError( - ErrCodeInvalidFVMStateAccessError, - "could not directly %s fvm internal state (address: %s: path: %s)", + ErrCodeInvalidInternalStateAccessError, + "could not directly %s flow internal state (%s)", opType, - address, - path) + id) } diff --git a/fvm/executionParameters.go b/fvm/executionParameters.go index 4e31c0fd3a9..1c349aef080 100644 --- a/fvm/executionParameters.go +++ b/fvm/executionParameters.go @@ -6,7 +6,6 @@ import ( "math" "github.com/onflow/cadence" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" "github.com/onflow/flow-go/fvm/blueprints" @@ -127,7 +126,7 @@ func (computer MeterParamOverridesComputer) getMeterParamOverrides( // Check that the service account exists because all the settings are // stored in it serviceAddress := computer.ctx.Chain.ServiceAddress() - service := runtime.Address(serviceAddress) + service := common.Address(serviceAddress) env := environment.NewScriptEnvironment( context.Background(), @@ -198,7 +197,7 @@ func (computer MeterParamOverridesComputer) getMeterParamOverrides( func getExecutionWeights[KindType common.ComputationKind | common.MemoryKind]( env environment.Environment, - service runtime.Address, + service common.Address, path cadence.Path, defaultWeights map[KindType]uint64, ) ( @@ -242,7 +241,7 @@ func getExecutionWeights[KindType common.ComputationKind | common.MemoryKind]( // GetExecutionEffortWeights reads stored execution effort weights from the service account func GetExecutionEffortWeights( env environment.Environment, - service runtime.Address, + service common.Address, ) ( computationWeights meter.ExecutionEffortWeights, err error, @@ -257,7 +256,7 @@ func GetExecutionEffortWeights( // GetExecutionMemoryWeights reads stored execution memory weights from the service account func GetExecutionMemoryWeights( env environment.Environment, - service runtime.Address, + service common.Address, ) ( memoryWeights meter.ExecutionMemoryWeights, err error, @@ -272,7 +271,7 @@ func GetExecutionMemoryWeights( // GetExecutionMemoryLimit reads the stored execution memory limit from the service account func GetExecutionMemoryLimit( env environment.Environment, - service runtime.Address, + service common.Address, ) ( memoryLimit uint64, err error, diff --git a/fvm/fvm_bench_test.go b/fvm/fvm_bench_test.go index 68de317f837..c27092aa615 100644 --- a/fvm/fvm_bench_test.go +++ b/fvm/fvm_bench_test.go @@ -226,7 +226,8 @@ func NewBasicBlockExecutor(tb testing.TB, chain flow.Chain, logger zerolog.Logge prov) require.NoError(tb, err) - view := delta.NewView(exeState.LedgerGetRegister(ledger, initialCommit)) + view := delta.NewDeltaView( + exeState.NewLedgerStorageSnapshot(ledger, initialCommit)) derivedChainData, err := derived.NewDerivedChainData( derived.DefaultDerivedDataCacheSize) @@ -262,9 +263,7 @@ func (b *BasicBlockExecutor) ExecuteCollections(tb testing.TB, collections [][]* computationResult, err := b.blockComputer.ExecuteBlock(context.Background(), executableBlock, b.activeView, derivedBlockData) require.NoError(tb, err) - endState, _, _, err := execution.GenerateExecutionResultAndChunkDataPacks(metrics.NewNoopCollector(), unittest.IdentifierFixture(), b.activeStateCommitment, computationResult) - require.NoError(tb, err) - b.activeStateCommitment = endState + b.activeStateCommitment = computationResult.EndState return computationResult } @@ -305,7 +304,8 @@ func (b *BasicBlockExecutor) SetupAccounts(tb testing.TB, privateKeys []flow.Acc if err != nil { tb.Fatal("setup account failed, error decoding events") } - addr = flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + addr = flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) break } } diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index 0c4e2152a34..ed40ab82de3 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -1571,7 +1571,8 @@ func TestBlockContext_GetAccount(t *testing.T) { // read the address of the account created (e.g. "0x01" and convert it to flow.address) data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) return address, privateKey.PublicKey(fvm.AccountKeyWeightThreshold).PublicKey } @@ -1661,7 +1662,7 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { num, err := strconv.ParseUint(tx.Logs[0], 10, 64) require.NoError(t, err) - require.Equal(t, uint64(0x8872445cb397f6d2), num) + require.Equal(t, uint64(0xde226d5af92d269), num) }) } @@ -1697,7 +1698,8 @@ func TestBlockContext_ExecuteTransaction_CreateAccount_WithMonotonicAddresses(t data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) assert.Equal(t, flow.HexToAddress("05"), address) } diff --git a/fvm/fvm_fuzz_test.go b/fvm/fvm_fuzz_test.go index fdd46d8553d..85881a474f0 100644 --- a/fvm/fvm_fuzz_test.go +++ b/fvm/fvm_fuzz_test.go @@ -272,7 +272,8 @@ func bootstrapFuzzStateAndTxContext(tb testing.TB) (bootstrappedVmTest, transact data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(tb, err) - address = flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address = flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) // ==== Transfer tokens to new account ==== txBody = transferTokensTx(chain). diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index 704bb7e7184..ec50613a2f3 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -36,9 +36,9 @@ var mainnetExecutionEffortWeights = meter.ExecutionEffortWeights{ common.ComputationKindStatement: 1569, common.ComputationKindLoop: 1569, common.ComputationKindFunctionInvocation: 1569, - meter.ComputationKindGetValue: 808, - meter.ComputationKindCreateAccount: 2837670, - meter.ComputationKindSetValue: 765, + environment.ComputationKindGetValue: 808, + environment.ComputationKindCreateAccount: 2837670, + environment.ComputationKindSetValue: 765, } type vmTest struct { @@ -980,7 +980,8 @@ func TestTransactionFeeDeduction(t *testing.T) { // read the address of the account created (e.g. "0x01" and convert it to flow.address) data, err := jsoncdc.Decode(nil, accountCreatedEvents[0].Payload) require.NoError(t, err) - address := flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address := flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) // ==== Transfer tokens to new account ==== txBody = transferTokensTx(chain). @@ -1490,10 +1491,13 @@ func TestStorageUsed(t *testing.T) { address, err := hex.DecodeString("2a3c4c2581cef731") require.NoError(t, err) + accountStatusId := flow.AccountStatusRegisterID( + flow.BytesToAddress(address)) + simpleView := utils.NewSimpleView() status := environment.NewAccountStatus() status.SetStorageUsed(5) - err = simpleView.Set(string(address), state.AccountStatusKey, status.ToBytes()) + err = simpleView.Set(accountStatusId, status.ToBytes()) require.NoError(t, err) script := fvm.Script(code) @@ -2035,7 +2039,8 @@ func TestInteractionLimit(t *testing.T) { if err != nil { return err } - address = flow.Address(data.(cadence.Event).Fields[0].(cadence.Address)) + address = flow.ConvertAddress( + data.(cadence.Event).Fields[0].(cadence.Address)) // ==== Transfer tokens to new account ==== txBody = transferTokensTx(chain). diff --git a/fvm/meter/computation_meter.go b/fvm/meter/computation_meter.go new file mode 100644 index 00000000000..e8fc14be658 --- /dev/null +++ b/fvm/meter/computation_meter.go @@ -0,0 +1,123 @@ +package meter + +import ( + "math" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/errors" +) + +// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/229 +// is merged and integrated. +const ( + ComputationKindCreateAccount = 2006 + ComputationKindGetValue = 2020 + ComputationKindSetValue = 2026 +) + +type MeteredComputationIntensities map[common.ComputationKind]uint + +var ( + // DefaultComputationWeights is the default weights for computation intensities + // these weighs make the computation metering the same as it was before dynamic execution fees + // these weighs make the computation metering the same as it was before dynamic execution fees + DefaultComputationWeights = ExecutionEffortWeights{ + common.ComputationKindStatement: 1 << MeterExecutionInternalPrecisionBytes, + common.ComputationKindLoop: 1 << MeterExecutionInternalPrecisionBytes, + common.ComputationKindFunctionInvocation: 1 << MeterExecutionInternalPrecisionBytes, + } +) + +// MeterExecutionInternalPrecisionBytes are the amount of bytes that are used internally by the +// WeigthedMeter to allow for metering computation smaller than one unit of computation. +// This allows for more fine weights. A weight of 1 unit of computation is equal to 1<<16. +// The minimum possible weight is 1/65536. +const MeterExecutionInternalPrecisionBytes = 16 + +type ExecutionEffortWeights map[common.ComputationKind]uint64 + +type ComputationMeterParameters struct { + computationLimit uint64 + computationWeights ExecutionEffortWeights +} + +func DefaultComputationMeterParameters() ComputationMeterParameters { + return ComputationMeterParameters{ + computationLimit: math.MaxUint64, + computationWeights: DefaultComputationWeights, + } +} + +func (params MeterParameters) WithComputationLimit(limit uint) MeterParameters { + newParams := params + newParams.computationLimit = uint64(limit) << MeterExecutionInternalPrecisionBytes + return newParams +} + +func (params MeterParameters) WithComputationWeights( + weights ExecutionEffortWeights, +) MeterParameters { + newParams := params + newParams.computationWeights = weights + return newParams +} + +func (params ComputationMeterParameters) ComputationWeights() ExecutionEffortWeights { + return params.computationWeights +} + +// TotalComputationLimit returns the total computation limit +func (params ComputationMeterParameters) TotalComputationLimit() uint { + return uint(params.computationLimit >> MeterExecutionInternalPrecisionBytes) +} + +type ComputationMeter struct { + params ComputationMeterParameters + + computationUsed uint64 + computationIntensities MeteredComputationIntensities +} + +func NewComputationMeter(params ComputationMeterParameters) ComputationMeter { + return ComputationMeter{ + params: params, + computationIntensities: make(MeteredComputationIntensities), + } +} + +// MeterComputation captures computation usage and returns an error if it goes beyond the limit +func (m *ComputationMeter) MeterComputation( + kind common.ComputationKind, + intensity uint, +) error { + m.computationIntensities[kind] += intensity + w, ok := m.params.computationWeights[kind] + if !ok { + return nil + } + m.computationUsed += w * uint64(intensity) + if m.computationUsed > m.params.computationLimit { + return errors.NewComputationLimitExceededError( + uint64(m.params.TotalComputationLimit())) + } + return nil +} + +// ComputationIntensities returns all the measured computational intensities +func (m *ComputationMeter) ComputationIntensities() MeteredComputationIntensities { + return m.computationIntensities +} + +// TotalComputationUsed returns the total computation used +func (m *ComputationMeter) TotalComputationUsed() uint64 { + return m.computationUsed >> MeterExecutionInternalPrecisionBytes +} + +func (m *ComputationMeter) Merge(child ComputationMeter) { + m.computationUsed = m.computationUsed + child.computationUsed + + for key, intensity := range child.computationIntensities { + m.computationIntensities[key] += intensity + } +} diff --git a/fvm/meter/event_meter.go b/fvm/meter/event_meter.go new file mode 100644 index 00000000000..ff4c604832e --- /dev/null +++ b/fvm/meter/event_meter.go @@ -0,0 +1,56 @@ +package meter + +import ( + "math" + + "github.com/onflow/flow-go/fvm/errors" +) + +type EventMeterParameters struct { + eventEmitByteLimit uint64 +} + +func DefaultEventMeterParameters() EventMeterParameters { + return EventMeterParameters{ + eventEmitByteLimit: math.MaxUint64, + } +} + +func (params MeterParameters) WithEventEmitByteLimit( + byteLimit uint64, +) MeterParameters { + newParams := params + newParams.eventEmitByteLimit = byteLimit + return newParams +} + +type EventMeter struct { + params EventMeterParameters + + totalEmittedEventBytes uint64 +} + +func NewEventMeter(params EventMeterParameters) EventMeter { + return EventMeter{ + params: params, + } +} + +func (m *EventMeter) MeterEmittedEvent(byteSize uint64) error { + m.totalEmittedEventBytes += byteSize + + if m.totalEmittedEventBytes > m.params.eventEmitByteLimit { + return errors.NewEventLimitExceededError( + m.totalEmittedEventBytes, + m.params.eventEmitByteLimit) + } + return nil +} + +func (m *EventMeter) TotalEmittedEventBytes() uint64 { + return m.totalEmittedEventBytes +} + +func (m *EventMeter) Merge(child EventMeter) { + m.totalEmittedEventBytes += child.TotalEmittedEventBytes() +} diff --git a/fvm/meter/interaction_meter.go b/fvm/meter/interaction_meter.go new file mode 100644 index 00000000000..c3c1e15a657 --- /dev/null +++ b/fvm/meter/interaction_meter.go @@ -0,0 +1,130 @@ +package meter + +import ( + "math" + + "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/model/flow" +) + +type MeteredStorageInteractionMap map[flow.RegisterID]uint64 + +type InteractionMeterParameters struct { + storageInteractionLimit uint64 +} + +func DefaultInteractionMeterParameters() InteractionMeterParameters { + return InteractionMeterParameters{ + storageInteractionLimit: math.MaxUint64, + } +} + +func (params MeterParameters) WithStorageInteractionLimit( + maxStorageInteractionLimit uint64, +) MeterParameters { + newParams := params + newParams.storageInteractionLimit = maxStorageInteractionLimit + return newParams +} + +type InteractionMeter struct { + params InteractionMeterParameters + + storageUpdateSizeMap map[flow.RegisterID]uint64 + totalStorageBytesRead uint64 + totalStorageBytesWritten uint64 +} + +func NewInteractionMeter(params InteractionMeterParameters) InteractionMeter { + return InteractionMeter{ + params: params, + storageUpdateSizeMap: make(map[flow.RegisterID]uint64), + } +} + +// MeterStorageRead captures storage read bytes count and returns an error +// if it goes beyond the total interaction limit and limit is enforced +func (m *InteractionMeter) MeterStorageRead( + storageKey flow.RegisterID, + value flow.RegisterValue, + enforceLimit bool, +) error { + + // all reads are on a View which only read from storage at the first read of a given key + if _, ok := m.storageUpdateSizeMap[storageKey]; !ok { + readByteSize := getStorageKeyValueSize(storageKey, value) + m.totalStorageBytesRead += readByteSize + m.storageUpdateSizeMap[storageKey] = readByteSize + } + + return m.checkStorageInteractionLimit(enforceLimit) +} + +// MeterStorageWrite captures storage written bytes count and returns an error +// if it goes beyond the total interaction limit and limit is enforced +func (m *InteractionMeter) MeterStorageWrite( + storageKey flow.RegisterID, + value flow.RegisterValue, + enforceLimit bool, +) error { + // all writes are on a View which only writes the latest updated value to storage at commit + if old, ok := m.storageUpdateSizeMap[storageKey]; ok { + m.totalStorageBytesWritten -= old + } + + updateSize := getStorageKeyValueSize(storageKey, value) + m.totalStorageBytesWritten += updateSize + m.storageUpdateSizeMap[storageKey] = updateSize + + return m.checkStorageInteractionLimit(enforceLimit) +} + +func (m *InteractionMeter) checkStorageInteractionLimit(enforceLimit bool) error { + if enforceLimit && + m.TotalBytesOfStorageInteractions() > m.params.storageInteractionLimit { + return errors.NewLedgerInteractionLimitExceededError( + m.TotalBytesOfStorageInteractions(), m.params.storageInteractionLimit) + } + return nil +} + +// TotalBytesReadFromStorage returns total number of byte read from storage +func (m *InteractionMeter) TotalBytesReadFromStorage() uint64 { + return m.totalStorageBytesRead +} + +// TotalBytesWrittenToStorage returns total number of byte written to storage +func (m *InteractionMeter) TotalBytesWrittenToStorage() uint64 { + return m.totalStorageBytesWritten +} + +// TotalBytesOfStorageInteractions returns total number of byte read and written from/to storage +func (m *InteractionMeter) TotalBytesOfStorageInteractions() uint64 { + return m.TotalBytesReadFromStorage() + m.TotalBytesWrittenToStorage() +} + +func getStorageKeyValueSize( + storageKey flow.RegisterID, + value flow.RegisterValue, +) uint64 { + return uint64(len(storageKey.Owner) + len(storageKey.Key) + len(value)) +} + +func GetStorageKeyValueSizeForTesting( + storageKey flow.RegisterID, + value flow.RegisterValue, +) uint64 { + return getStorageKeyValueSize(storageKey, value) +} + +func (m *InteractionMeter) GetStorageUpdateSizeMapForTesting() MeteredStorageInteractionMap { + return m.storageUpdateSizeMap +} + +func (m *InteractionMeter) Merge(child InteractionMeter) { + for key, value := range child.storageUpdateSizeMap { + m.storageUpdateSizeMap[key] = value + } + m.totalStorageBytesRead += child.TotalBytesReadFromStorage() + m.totalStorageBytesWritten += child.TotalBytesWrittenToStorage() +} diff --git a/fvm/meter/memory_meter.go b/fvm/meter/memory_meter.go new file mode 100644 index 00000000000..9519cfd77c7 --- /dev/null +++ b/fvm/meter/memory_meter.go @@ -0,0 +1,323 @@ +package meter + +import ( + "math" + + "github.com/onflow/cadence/runtime/common" + + "github.com/onflow/flow-go/fvm/errors" +) + +var ( + // DefaultMemoryWeights are currently hard-coded here. In the future we might like to + // define this in a contract similar to the computation weights + DefaultMemoryWeights = ExecutionMemoryWeights{ + + // Values + + common.MemoryKindAddressValue: 32, + common.MemoryKindStringValue: 138, + common.MemoryKindCharacterValue: 24, + common.MemoryKindNumberValue: 8, + // weights for these values include the cost of the Go struct itself (first number) + // as well as the overhead for creation of the underlying atree (second number) + common.MemoryKindArrayValueBase: 57 + 48, + common.MemoryKindDictionaryValueBase: 33 + 96, + common.MemoryKindCompositeValueBase: 233 + 96, + common.MemoryKindSimpleCompositeValue: 73, + common.MemoryKindSimpleCompositeValueBase: 89, + common.MemoryKindOptionalValue: 41, + common.MemoryKindTypeValue: 17, + common.MemoryKindPathValue: 24, + common.MemoryKindStorageCapabilityValue: 1, + common.MemoryKindPathLinkValue: 1, + common.MemoryKindAccountLinkValue: 1, + common.MemoryKindAccountReferenceValue: 1, + common.MemoryKindPublishedValue: 1, + common.MemoryKindStorageReferenceValue: 41, + common.MemoryKindEphemeralReferenceValue: 41, + common.MemoryKindInterpretedFunctionValue: 128, + common.MemoryKindHostFunctionValue: 41, + common.MemoryKindBoundFunctionValue: 25, + common.MemoryKindBigInt: 50, + common.MemoryKindVoidExpression: 1, + + // Atree + + common.MemoryKindAtreeArrayDataSlab: 80, + common.MemoryKindAtreeArrayMetaDataSlab: 1024, + common.MemoryKindAtreeArrayElementOverhead: 16, + common.MemoryKindAtreeMapDataSlab: 144, + common.MemoryKindAtreeMapMetaDataSlab: 1024, + common.MemoryKindAtreeMapElementOverhead: 64, + common.MemoryKindAtreeMapPreAllocatedElement: 24, + common.MemoryKindAtreeEncodedSlab: 1536, + + // Static Types + + common.MemoryKindPrimitiveStaticType: 8, + common.MemoryKindCompositeStaticType: 17, + common.MemoryKindInterfaceStaticType: 17, + common.MemoryKindVariableSizedStaticType: 17, + common.MemoryKindConstantSizedStaticType: 25, + common.MemoryKindDictionaryStaticType: 33, + common.MemoryKindOptionalStaticType: 17, + common.MemoryKindRestrictedStaticType: 41, + common.MemoryKindReferenceStaticType: 41, + common.MemoryKindCapabilityStaticType: 17, + common.MemoryKindFunctionStaticType: 9, + + // Cadence Values + + common.MemoryKindCadenceVoidValue: 1, + common.MemoryKindCadenceOptionalValue: 17, + common.MemoryKindCadenceBoolValue: 8, + common.MemoryKindCadenceStringValue: 16, + common.MemoryKindCadenceCharacterValue: 16, + common.MemoryKindCadenceAddressValue: 8, + common.MemoryKindCadenceIntValue: 50, + common.MemoryKindCadenceNumberValue: 1, + common.MemoryKindCadenceArrayValueBase: 41, + common.MemoryKindCadenceArrayValueLength: 16, + common.MemoryKindCadenceDictionaryValue: 41, + common.MemoryKindCadenceKeyValuePair: 33, + common.MemoryKindCadenceStructValueBase: 33, + common.MemoryKindCadenceStructValueSize: 16, + common.MemoryKindCadenceResourceValueBase: 33, + common.MemoryKindCadenceResourceValueSize: 16, + common.MemoryKindCadenceEventValueBase: 33, + common.MemoryKindCadenceEventValueSize: 16, + common.MemoryKindCadenceContractValueBase: 33, + common.MemoryKindCadenceContractValueSize: 16, + common.MemoryKindCadenceEnumValueBase: 33, + common.MemoryKindCadenceEnumValueSize: 16, + common.MemoryKindCadencePathLinkValue: 1, + common.MemoryKindCadencePathValue: 33, + common.MemoryKindCadenceTypeValue: 17, + common.MemoryKindCadenceStorageCapabilityValue: 1, + common.MemoryKindCadenceFunctionValue: 1, + + // Cadence Types + + common.MemoryKindCadenceSimpleType: 1, + common.MemoryKindCadenceOptionalType: 17, + common.MemoryKindCadenceVariableSizedArrayType: 17, + common.MemoryKindCadenceConstantSizedArrayType: 25, + common.MemoryKindCadenceDictionaryType: 33, + common.MemoryKindCadenceField: 33, + common.MemoryKindCadenceParameter: 49, + common.MemoryKindCadenceStructType: 81, + common.MemoryKindCadenceResourceType: 81, + common.MemoryKindCadenceEventType: 81, + common.MemoryKindCadenceContractType: 81, + common.MemoryKindCadenceStructInterfaceType: 81, + common.MemoryKindCadenceResourceInterfaceType: 81, + common.MemoryKindCadenceContractInterfaceType: 81, + common.MemoryKindCadenceFunctionType: 41, + common.MemoryKindCadenceReferenceType: 25, + common.MemoryKindCadenceRestrictedType: 57, + common.MemoryKindCadenceCapabilityType: 17, + common.MemoryKindCadenceEnumType: 97, + + // Misc + + common.MemoryKindRawString: 9, + common.MemoryKindAddressLocation: 18, + common.MemoryKindBytes: 24, + common.MemoryKindVariable: 18, + common.MemoryKindCompositeTypeInfo: 41, + common.MemoryKindCompositeField: 33, + common.MemoryKindInvocation: 89, + common.MemoryKindStorageMap: 58, + common.MemoryKindStorageKey: 41, + + // Tokens + + common.MemoryKindErrorToken: 41, + common.MemoryKindTypeToken: 25, + common.MemoryKindSpaceToken: 50, + + // AST nodes + + common.MemoryKindProgram: 220, + common.MemoryKindIdentifier: 17, + common.MemoryKindArgument: 49, + common.MemoryKindBlock: 25, + common.MemoryKindFunctionBlock: 25, + common.MemoryKindParameter: 25, + common.MemoryKindParameterList: 59, + common.MemoryKindTransfer: 1, + common.MemoryKindMembers: 276, + common.MemoryKindTypeAnnotation: 25, + common.MemoryKindDictionaryEntry: 33, + + common.MemoryKindFunctionDeclaration: 49, + common.MemoryKindCompositeDeclaration: 65, + common.MemoryKindInterfaceDeclaration: 41, + common.MemoryKindEnumCaseDeclaration: 25, + common.MemoryKindFieldDeclaration: 41, + common.MemoryKindTransactionDeclaration: 81, + common.MemoryKindImportDeclaration: 41, + common.MemoryKindVariableDeclaration: 97, + common.MemoryKindSpecialFunctionDeclaration: 17, + common.MemoryKindPragmaDeclaration: 17, + + common.MemoryKindAssignmentStatement: 41, + common.MemoryKindBreakStatement: 1, + common.MemoryKindContinueStatement: 1, + common.MemoryKindEmitStatement: 9, + common.MemoryKindExpressionStatement: 17, + common.MemoryKindForStatement: 33, + common.MemoryKindIfStatement: 33, + common.MemoryKindReturnStatement: 17, + common.MemoryKindSwapStatement: 33, + common.MemoryKindSwitchStatement: 41, + common.MemoryKindWhileStatement: 25, + + common.MemoryKindBooleanExpression: 9, + common.MemoryKindNilExpression: 1, + common.MemoryKindStringExpression: 17, + common.MemoryKindIntegerExpression: 33, + common.MemoryKindFixedPointExpression: 49, + common.MemoryKindArrayExpression: 25, + common.MemoryKindDictionaryExpression: 25, + common.MemoryKindIdentifierExpression: 1, + common.MemoryKindInvocationExpression: 49, + common.MemoryKindMemberExpression: 25, + common.MemoryKindIndexExpression: 33, + common.MemoryKindConditionalExpression: 49, + common.MemoryKindUnaryExpression: 25, + common.MemoryKindBinaryExpression: 41, + common.MemoryKindFunctionExpression: 25, + common.MemoryKindCastingExpression: 41, + common.MemoryKindCreateExpression: 9, + common.MemoryKindDestroyExpression: 17, + common.MemoryKindReferenceExpression: 33, + common.MemoryKindForceExpression: 17, + common.MemoryKindPathExpression: 1, + + common.MemoryKindConstantSizedType: 25, + common.MemoryKindDictionaryType: 33, + common.MemoryKindFunctionType: 33, + common.MemoryKindInstantiationType: 41, + common.MemoryKindNominalType: 25, + common.MemoryKindOptionalType: 17, + common.MemoryKindReferenceType: 25, + common.MemoryKindRestrictedType: 41, + common.MemoryKindVariableSizedType: 17, + + common.MemoryKindPosition: 25, + common.MemoryKindRange: 1, + common.MemoryKindActivation: 128, + common.MemoryKindActivationEntries: 256, + common.MemoryKindElaboration: 501, + + // sema types + common.MemoryKindVariableSizedSemaType: 51, + common.MemoryKindConstantSizedSemaType: 59, + common.MemoryKindDictionarySemaType: 67, + common.MemoryKindOptionalSemaType: 17, + common.MemoryKindRestrictedSemaType: 75, + common.MemoryKindReferenceSemaType: 25, + common.MemoryKindCapabilitySemaType: 51, + + // ordered-map + common.MemoryKindOrderedMap: 17, + common.MemoryKindOrderedMapEntryList: 50, + common.MemoryKindOrderedMapEntry: 64, + } +) + +func _() { + // A compiler error signifies that we have not accounted for all memory kinds + var x [1]struct{} + _ = x[int(common.MemoryKindLast)-len(DefaultMemoryWeights)-1] +} + +type ExecutionMemoryWeights map[common.MemoryKind]uint64 +type MeteredMemoryIntensities map[common.MemoryKind]uint + +type MemoryMeterParameters struct { + memoryLimit uint64 + memoryWeights ExecutionMemoryWeights +} + +func DefaultMemoryParameters() MemoryMeterParameters { + return MemoryMeterParameters{ + memoryLimit: math.MaxUint64, + memoryWeights: DefaultMemoryWeights, + } +} + +func (params MemoryMeterParameters) MemoryWeights() ExecutionMemoryWeights { + return params.memoryWeights +} + +// TotalMemoryLimit returns the total memory limit +func (params MemoryMeterParameters) TotalMemoryLimit() uint64 { + return params.memoryLimit +} + +func (params MeterParameters) WithMemoryLimit(limit uint64) MeterParameters { + newParams := params + newParams.memoryLimit = limit + return newParams +} + +func (params MeterParameters) WithMemoryWeights( + weights ExecutionMemoryWeights, +) MeterParameters { + newParams := params + newParams.memoryWeights = weights + return newParams +} + +type MemoryMeter struct { + params MemoryMeterParameters + + memoryIntensities MeteredMemoryIntensities + memoryEstimate uint64 +} + +// MemoryIntensities returns all the measured memory intensities +func (m *MemoryMeter) MemoryIntensities() MeteredMemoryIntensities { + return m.memoryIntensities +} + +// NewMemoryMeter constructs a new Meter +func NewMemoryMeter(params MemoryMeterParameters) MemoryMeter { + m := MemoryMeter{ + params: params, + memoryIntensities: make(MeteredMemoryIntensities), + } + + return m +} + +// MeterMemory captures memory usage and returns an error if it goes beyond the limit +func (m *MemoryMeter) MeterMemory(kind common.MemoryKind, intensity uint) error { + m.memoryIntensities[kind] += intensity + w, ok := m.params.memoryWeights[kind] + if !ok { + return nil + } + m.memoryEstimate += w * uint64(intensity) + if m.memoryEstimate > m.params.memoryLimit { + return errors.NewMemoryLimitExceededError(m.params.TotalMemoryLimit()) + } + return nil +} + +// TotalMemoryEstimate returns the total memory used +func (m *MemoryMeter) TotalMemoryEstimate() uint64 { + return m.memoryEstimate +} + +// Merge merges the input meter into the current meter and checks for the limits +func (m *MemoryMeter) Merge(child MemoryMeter) { + m.memoryEstimate = m.memoryEstimate + child.TotalMemoryEstimate() + + for key, intensity := range child.memoryIntensities { + m.memoryIntensities[key] += intensity + } +} diff --git a/fvm/meter/meter.go b/fvm/meter/meter.go index bc96447dd1c..2654a03aaa7 100644 --- a/fvm/meter/meter.go +++ b/fvm/meter/meter.go @@ -1,553 +1,52 @@ package meter -import ( - "math" - - "github.com/onflow/cadence/runtime/common" - - "github.com/onflow/flow-go/fvm/errors" - "github.com/onflow/flow-go/model/flow" -) - -// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/229 -// is merged and integrated. -const ( - ComputationKindCreateAccount = 2006 - ComputationKindGetValue = 2020 - ComputationKindSetValue = 2026 -) - -type MeteredComputationIntensities map[common.ComputationKind]uint -type MeteredMemoryIntensities map[common.MemoryKind]uint -type MeteredStorageInteractionMap map[StorageInteractionKey]uint64 - -type StorageInteractionKey struct { - Owner, Key string -} - -// MeterExecutionInternalPrecisionBytes are the amount of bytes that are used internally by the WeigthedMeter -// to allow for metering computation smaller than one unit of computation. This allows for more fine weights. -// A weight of 1 unit of computation is equal to 1<<16. The minimum possible weight is 1/65536. -const MeterExecutionInternalPrecisionBytes = 16 - -type ExecutionEffortWeights map[common.ComputationKind]uint64 -type ExecutionMemoryWeights map[common.MemoryKind]uint64 - -var ( - // DefaultComputationWeights is the default weights for computation intensities - // these weighs make the computation metering the same as it was before dynamic execution fees - DefaultComputationWeights = ExecutionEffortWeights{ - common.ComputationKindStatement: 1 << MeterExecutionInternalPrecisionBytes, - common.ComputationKindLoop: 1 << MeterExecutionInternalPrecisionBytes, - common.ComputationKindFunctionInvocation: 1 << MeterExecutionInternalPrecisionBytes, - } - - // DefaultMemoryWeights are currently hard-coded here. In the future we might like to - // define this in a contract similar to the computation weights - DefaultMemoryWeights = ExecutionMemoryWeights{ - - // Values - - common.MemoryKindAddressValue: 32, - common.MemoryKindStringValue: 138, - common.MemoryKindCharacterValue: 24, - common.MemoryKindNumberValue: 8, - // weights for these values include the cost of the Go struct itself (first number) - // as well as the overhead for creation of the underlying atree (second number) - common.MemoryKindArrayValueBase: 57 + 48, - common.MemoryKindDictionaryValueBase: 33 + 96, - common.MemoryKindCompositeValueBase: 233 + 96, - common.MemoryKindSimpleCompositeValue: 73, - common.MemoryKindSimpleCompositeValueBase: 89, - common.MemoryKindOptionalValue: 41, - common.MemoryKindTypeValue: 17, - common.MemoryKindPathValue: 24, - common.MemoryKindStorageCapabilityValue: 1, - common.MemoryKindPathLinkValue: 1, - common.MemoryKindAccountLinkValue: 1, - common.MemoryKindAccountReferenceValue: 1, - common.MemoryKindPublishedValue: 1, - common.MemoryKindStorageReferenceValue: 41, - common.MemoryKindEphemeralReferenceValue: 41, - common.MemoryKindInterpretedFunctionValue: 128, - common.MemoryKindHostFunctionValue: 41, - common.MemoryKindBoundFunctionValue: 25, - common.MemoryKindBigInt: 50, - common.MemoryKindVoidExpression: 1, - - // Atree - - common.MemoryKindAtreeArrayDataSlab: 80, - common.MemoryKindAtreeArrayMetaDataSlab: 1024, - common.MemoryKindAtreeArrayElementOverhead: 16, - common.MemoryKindAtreeMapDataSlab: 144, - common.MemoryKindAtreeMapMetaDataSlab: 1024, - common.MemoryKindAtreeMapElementOverhead: 64, - common.MemoryKindAtreeMapPreAllocatedElement: 24, - common.MemoryKindAtreeEncodedSlab: 1536, - - // Static Types - - common.MemoryKindPrimitiveStaticType: 8, - common.MemoryKindCompositeStaticType: 17, - common.MemoryKindInterfaceStaticType: 17, - common.MemoryKindVariableSizedStaticType: 17, - common.MemoryKindConstantSizedStaticType: 25, - common.MemoryKindDictionaryStaticType: 33, - common.MemoryKindOptionalStaticType: 17, - common.MemoryKindRestrictedStaticType: 41, - common.MemoryKindReferenceStaticType: 41, - common.MemoryKindCapabilityStaticType: 17, - common.MemoryKindFunctionStaticType: 9, - - // Cadence Values - - common.MemoryKindCadenceVoidValue: 1, - common.MemoryKindCadenceOptionalValue: 17, - common.MemoryKindCadenceBoolValue: 8, - common.MemoryKindCadenceStringValue: 16, - common.MemoryKindCadenceCharacterValue: 16, - common.MemoryKindCadenceAddressValue: 8, - common.MemoryKindCadenceIntValue: 50, - common.MemoryKindCadenceNumberValue: 1, - common.MemoryKindCadenceArrayValueBase: 41, - common.MemoryKindCadenceArrayValueLength: 16, - common.MemoryKindCadenceDictionaryValue: 41, - common.MemoryKindCadenceKeyValuePair: 33, - common.MemoryKindCadenceStructValueBase: 33, - common.MemoryKindCadenceStructValueSize: 16, - common.MemoryKindCadenceResourceValueBase: 33, - common.MemoryKindCadenceResourceValueSize: 16, - common.MemoryKindCadenceEventValueBase: 33, - common.MemoryKindCadenceEventValueSize: 16, - common.MemoryKindCadenceContractValueBase: 33, - common.MemoryKindCadenceContractValueSize: 16, - common.MemoryKindCadenceEnumValueBase: 33, - common.MemoryKindCadenceEnumValueSize: 16, - common.MemoryKindCadencePathLinkValue: 1, - common.MemoryKindCadencePathValue: 33, - common.MemoryKindCadenceTypeValue: 17, - common.MemoryKindCadenceStorageCapabilityValue: 1, - common.MemoryKindCadenceFunctionValue: 1, - - // Cadence Types - - common.MemoryKindCadenceSimpleType: 1, - common.MemoryKindCadenceOptionalType: 17, - common.MemoryKindCadenceVariableSizedArrayType: 17, - common.MemoryKindCadenceConstantSizedArrayType: 25, - common.MemoryKindCadenceDictionaryType: 33, - common.MemoryKindCadenceField: 33, - common.MemoryKindCadenceParameter: 49, - common.MemoryKindCadenceStructType: 81, - common.MemoryKindCadenceResourceType: 81, - common.MemoryKindCadenceEventType: 81, - common.MemoryKindCadenceContractType: 81, - common.MemoryKindCadenceStructInterfaceType: 81, - common.MemoryKindCadenceResourceInterfaceType: 81, - common.MemoryKindCadenceContractInterfaceType: 81, - common.MemoryKindCadenceFunctionType: 41, - common.MemoryKindCadenceReferenceType: 25, - common.MemoryKindCadenceRestrictedType: 57, - common.MemoryKindCadenceCapabilityType: 17, - common.MemoryKindCadenceEnumType: 97, - - // Misc - - common.MemoryKindRawString: 9, - common.MemoryKindAddressLocation: 18, - common.MemoryKindBytes: 24, - common.MemoryKindVariable: 18, - common.MemoryKindCompositeTypeInfo: 41, - common.MemoryKindCompositeField: 33, - common.MemoryKindInvocation: 89, - common.MemoryKindStorageMap: 58, - common.MemoryKindStorageKey: 41, - - // Tokens - - common.MemoryKindErrorToken: 41, - common.MemoryKindTypeToken: 25, - common.MemoryKindSpaceToken: 50, - - // AST nodes - - common.MemoryKindProgram: 220, - common.MemoryKindIdentifier: 17, - common.MemoryKindArgument: 49, - common.MemoryKindBlock: 25, - common.MemoryKindFunctionBlock: 25, - common.MemoryKindParameter: 25, - common.MemoryKindParameterList: 59, - common.MemoryKindTransfer: 1, - common.MemoryKindMembers: 276, - common.MemoryKindTypeAnnotation: 25, - common.MemoryKindDictionaryEntry: 33, - - common.MemoryKindFunctionDeclaration: 49, - common.MemoryKindCompositeDeclaration: 65, - common.MemoryKindInterfaceDeclaration: 41, - common.MemoryKindEnumCaseDeclaration: 25, - common.MemoryKindFieldDeclaration: 41, - common.MemoryKindTransactionDeclaration: 81, - common.MemoryKindImportDeclaration: 41, - common.MemoryKindVariableDeclaration: 97, - common.MemoryKindSpecialFunctionDeclaration: 17, - common.MemoryKindPragmaDeclaration: 17, - - common.MemoryKindAssignmentStatement: 41, - common.MemoryKindBreakStatement: 1, - common.MemoryKindContinueStatement: 1, - common.MemoryKindEmitStatement: 9, - common.MemoryKindExpressionStatement: 17, - common.MemoryKindForStatement: 33, - common.MemoryKindIfStatement: 33, - common.MemoryKindReturnStatement: 17, - common.MemoryKindSwapStatement: 33, - common.MemoryKindSwitchStatement: 41, - common.MemoryKindWhileStatement: 25, - - common.MemoryKindBooleanExpression: 9, - common.MemoryKindNilExpression: 1, - common.MemoryKindStringExpression: 17, - common.MemoryKindIntegerExpression: 33, - common.MemoryKindFixedPointExpression: 49, - common.MemoryKindArrayExpression: 25, - common.MemoryKindDictionaryExpression: 25, - common.MemoryKindIdentifierExpression: 1, - common.MemoryKindInvocationExpression: 49, - common.MemoryKindMemberExpression: 25, - common.MemoryKindIndexExpression: 33, - common.MemoryKindConditionalExpression: 49, - common.MemoryKindUnaryExpression: 25, - common.MemoryKindBinaryExpression: 41, - common.MemoryKindFunctionExpression: 25, - common.MemoryKindCastingExpression: 41, - common.MemoryKindCreateExpression: 9, - common.MemoryKindDestroyExpression: 17, - common.MemoryKindReferenceExpression: 33, - common.MemoryKindForceExpression: 17, - common.MemoryKindPathExpression: 1, - - common.MemoryKindConstantSizedType: 25, - common.MemoryKindDictionaryType: 33, - common.MemoryKindFunctionType: 33, - common.MemoryKindInstantiationType: 41, - common.MemoryKindNominalType: 25, - common.MemoryKindOptionalType: 17, - common.MemoryKindReferenceType: 25, - common.MemoryKindRestrictedType: 41, - common.MemoryKindVariableSizedType: 17, - - common.MemoryKindPosition: 25, - common.MemoryKindRange: 1, - common.MemoryKindActivation: 128, - common.MemoryKindActivationEntries: 256, - common.MemoryKindElaboration: 501, - - // sema types - common.MemoryKindVariableSizedSemaType: 51, - common.MemoryKindConstantSizedSemaType: 59, - common.MemoryKindDictionarySemaType: 67, - common.MemoryKindOptionalSemaType: 17, - common.MemoryKindRestrictedSemaType: 75, - common.MemoryKindReferenceSemaType: 25, - common.MemoryKindCapabilitySemaType: 51, - - // ordered-map - common.MemoryKindOrderedMap: 17, - common.MemoryKindOrderedMapEntryList: 50, - common.MemoryKindOrderedMapEntry: 64, - } -) - -func _() { - // A compiler error signifies that we have not accounted for all memory kinds - var x [1]struct{} - _ = x[int(common.MemoryKindLast)-len(DefaultMemoryWeights)-1] -} - type MeterParameters struct { - computationLimit uint64 - computationWeights ExecutionEffortWeights - - memoryLimit uint64 - memoryWeights ExecutionMemoryWeights - - storageInteractionLimit uint64 - eventEmitByteLimit uint64 + ComputationMeterParameters + MemoryMeterParameters + EventMeterParameters + InteractionMeterParameters } func DefaultParameters() MeterParameters { - // This is needed to work around golang's compiler bug - umax := uint(math.MaxUint) return MeterParameters{ - computationLimit: uint64(umax) << MeterExecutionInternalPrecisionBytes, - memoryLimit: math.MaxUint64, - computationWeights: DefaultComputationWeights, - memoryWeights: DefaultMemoryWeights, - storageInteractionLimit: math.MaxUint64, - eventEmitByteLimit: math.MaxUint64, + ComputationMeterParameters: DefaultComputationMeterParameters(), + MemoryMeterParameters: DefaultMemoryParameters(), + EventMeterParameters: DefaultEventMeterParameters(), + InteractionMeterParameters: DefaultInteractionMeterParameters(), } } -func (params MeterParameters) WithComputationLimit(limit uint) MeterParameters { - newParams := params - newParams.computationLimit = uint64(limit) << MeterExecutionInternalPrecisionBytes - return newParams -} - -func (params MeterParameters) WithComputationWeights( - weights ExecutionEffortWeights, -) MeterParameters { - newParams := params - newParams.computationWeights = weights - return newParams -} - -func (params MeterParameters) WithMemoryLimit(limit uint64) MeterParameters { - newParams := params - newParams.memoryLimit = limit - return newParams -} - -func (params MeterParameters) WithMemoryWeights( - weights ExecutionMemoryWeights, -) MeterParameters { - newParams := params - newParams.memoryWeights = weights - return newParams -} - -func (params MeterParameters) WithStorageInteractionLimit( - maxStorageInteractionLimit uint64, -) MeterParameters { - newParams := params - newParams.storageInteractionLimit = maxStorageInteractionLimit - return newParams -} - -func (params MeterParameters) WithEventEmitByteLimit( - byteLimit uint64, -) MeterParameters { - newParams := params - newParams.eventEmitByteLimit = byteLimit - return newParams -} - -func (params MeterParameters) ComputationWeights() ExecutionEffortWeights { - return params.computationWeights -} - -// TotalComputationLimit returns the total computation limit -func (params MeterParameters) TotalComputationLimit() uint { - return uint(params.computationLimit >> MeterExecutionInternalPrecisionBytes) -} - -func (params MeterParameters) MemoryWeights() ExecutionMemoryWeights { - return params.memoryWeights -} - -// TotalMemoryLimit returns the total memory limit -func (params MeterParameters) TotalMemoryLimit() uint64 { - return params.memoryLimit -} - // Meter collects memory and computation usage and enforces limits // for any each memory/computation usage call it sums intensity multiplied by the weight of the intensity to the total // memory/computation usage metrics and returns error if limits are not met. type Meter struct { MeterParameters - computationUsed uint64 - memoryEstimate uint64 - - computationIntensities MeteredComputationIntensities - memoryIntensities MeteredMemoryIntensities - - storageUpdateSizeMap map[StorageInteractionKey]uint64 - totalStorageBytesRead uint64 - totalStorageBytesWritten uint64 - - totalEmittedEventBytes uint64 + MemoryMeter + ComputationMeter + EventMeter + InteractionMeter } -type MeterOptions func(*Meter) - // NewMeter constructs a new Meter func NewMeter(params MeterParameters) *Meter { - m := &Meter{ - MeterParameters: params, - computationIntensities: make(MeteredComputationIntensities), - memoryIntensities: make(MeteredMemoryIntensities), - storageUpdateSizeMap: make(MeteredStorageInteractionMap), + return &Meter{ + MeterParameters: params, + ComputationMeter: NewComputationMeter(params.ComputationMeterParameters), + MemoryMeter: NewMemoryMeter(params.MemoryMeterParameters), + EventMeter: NewEventMeter(params.EventMeterParameters), + InteractionMeter: NewInteractionMeter(params.InteractionMeterParameters), } - - return m } -// MergeMeter merges the input meter into the current meter and checks for the limits +// MergeMeter merges the input meter into the current meter func (m *Meter) MergeMeter(child *Meter) { if child == nil { return } - var childComputationUsed = child.computationUsed - m.computationUsed = m.computationUsed + childComputationUsed - - for key, intensity := range child.computationIntensities { - m.computationIntensities[key] += intensity - } - - m.memoryEstimate = m.memoryEstimate + child.TotalMemoryEstimate() - - for key, intensity := range child.memoryIntensities { - m.memoryIntensities[key] += intensity - } - - // merge storage meters - for key, value := range child.storageUpdateSizeMap { - m.storageUpdateSizeMap[key] = value - } - m.totalStorageBytesRead += child.TotalBytesReadFromStorage() - m.totalStorageBytesWritten += child.TotalBytesWrittenToStorage() - - m.totalEmittedEventBytes += child.TotalEmittedEventBytes() -} - -// MeterComputation captures computation usage and returns an error if it goes beyond the limit -func (m *Meter) MeterComputation(kind common.ComputationKind, intensity uint) error { - m.computationIntensities[kind] += intensity - w, ok := m.computationWeights[kind] - if !ok { - return nil - } - m.computationUsed += w * uint64(intensity) - if m.computationUsed > m.computationLimit { - return errors.NewComputationLimitExceededError(uint64(m.TotalComputationLimit())) - } - return nil -} - -// ComputationIntensities returns all the measured computational intensities -func (m *Meter) ComputationIntensities() MeteredComputationIntensities { - return m.computationIntensities -} - -// TotalComputationUsed returns the total computation used -func (m *Meter) TotalComputationUsed() uint64 { - return m.computationUsed >> MeterExecutionInternalPrecisionBytes -} - -// MeterMemory captures memory usage and returns an error if it goes beyond the limit -func (m *Meter) MeterMemory(kind common.MemoryKind, intensity uint) error { - m.memoryIntensities[kind] += intensity - w, ok := m.memoryWeights[kind] - if !ok { - return nil - } - m.memoryEstimate += w * uint64(intensity) - if m.memoryEstimate > m.memoryLimit { - return errors.NewMemoryLimitExceededError(uint64(m.TotalMemoryLimit())) - } - return nil -} - -// MemoryIntensities returns all the measured memory intensities -func (m *Meter) MemoryIntensities() MeteredMemoryIntensities { - return m.memoryIntensities -} - -// TotalMemoryEstimate returns the total memory used -func (m *Meter) TotalMemoryEstimate() uint64 { - return m.memoryEstimate -} - -// MeterStorageRead captures storage read bytes count and returns an error -// if it goes beyond the total interaction limit and limit is enforced -func (m *Meter) MeterStorageRead( - storageKey StorageInteractionKey, - value flow.RegisterValue, - enforceLimit bool) error { - - // all reads are on a View which only read from storage at the first read of a given key - if _, ok := m.storageUpdateSizeMap[storageKey]; !ok { - readByteSize := getStorageKeyValueSize(storageKey, value) - m.totalStorageBytesRead += readByteSize - m.storageUpdateSizeMap[storageKey] = readByteSize - } - - return m.checkStorageInteractionLimit(enforceLimit) -} - -// MeterStorageRead captures storage written bytes count and returns an error -// if it goes beyond the total interaction limit and limit is enforced -func (m *Meter) MeterStorageWrite( - storageKey StorageInteractionKey, - value flow.RegisterValue, - enforceLimit bool) error { - // all writes are on a View which only writes the latest updated value to storage at commit - if old, ok := m.storageUpdateSizeMap[storageKey]; ok { - m.totalStorageBytesWritten -= old - } - - updateSize := getStorageKeyValueSize(storageKey, value) - m.totalStorageBytesWritten += updateSize - m.storageUpdateSizeMap[storageKey] = updateSize - - return m.checkStorageInteractionLimit(enforceLimit) -} - -func (m *Meter) checkStorageInteractionLimit(enforceLimit bool) error { - if enforceLimit && - m.TotalBytesOfStorageInteractions() > m.storageInteractionLimit { - return errors.NewLedgerInteractionLimitExceededError( - m.TotalBytesOfStorageInteractions(), m.storageInteractionLimit) - } - return nil -} - -// TotalBytesReadFromStorage returns total number of byte read from storage -func (m *Meter) TotalBytesReadFromStorage() uint64 { - return m.totalStorageBytesRead -} - -// TotalBytesReadFromStorage returns total number of byte written to storage -func (m *Meter) TotalBytesWrittenToStorage() uint64 { - return m.totalStorageBytesWritten -} - -// TotalBytesOfStorageInteractions returns total number of byte read and written from/to storage -func (m *Meter) TotalBytesOfStorageInteractions() uint64 { - return m.TotalBytesReadFromStorage() + m.TotalBytesWrittenToStorage() -} - -func getStorageKeyValueSize(storageKey StorageInteractionKey, - value flow.RegisterValue) uint64 { - return uint64(len(storageKey.Owner) + len(storageKey.Key) + len(value)) -} - -func GetStorageKeyValueSizeForTesting( - storageKey StorageInteractionKey, - value flow.RegisterValue) uint64 { - return getStorageKeyValueSize(storageKey, value) -} - -func (m *Meter) GetStorageUpdateSizeMapForTesting() MeteredStorageInteractionMap { - return m.storageUpdateSizeMap -} - -func (m *Meter) MeterEmittedEvent(byteSize uint64) error { - m.totalEmittedEventBytes += uint64(byteSize) - - if m.totalEmittedEventBytes > m.eventEmitByteLimit { - return errors.NewEventLimitExceededError( - m.totalEmittedEventBytes, - m.eventEmitByteLimit) - } - return nil -} - -func (m *Meter) TotalEmittedEventBytes() uint64 { - return m.totalEmittedEventBytes + m.ComputationMeter.Merge(child.ComputationMeter) + m.MemoryMeter.Merge(child.MemoryMeter) + m.EventMeter.Merge(child.EventMeter) + m.InteractionMeter.Merge(child.InteractionMeter) } diff --git a/fvm/meter/meter_test.go b/fvm/meter/meter_test.go index 7f84c5e3ff8..896c2ea3227 100644 --- a/fvm/meter/meter_test.go +++ b/fvm/meter/meter_test.go @@ -12,6 +12,7 @@ import ( "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" + "github.com/onflow/flow-go/model/flow" ) func TestWeightedComputationMetering(t *testing.T) { @@ -383,7 +384,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters(), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} size1 := meter.GetStorageKeyValueSizeForTesting(key1, val1) @@ -398,7 +399,7 @@ func TestStorageLimits(t *testing.T) { require.Equal(t, meter1.TotalBytesReadFromStorage(), size1) // first read of key2 - key2 := meter.StorageInteractionKey{Owner: "", Key: "2"} + key2 := flow.NewRegisterID("", "2") val2 := []byte{0x3, 0x2, 0x1} size2 := meter.GetStorageKeyValueSizeForTesting(key2, val2) @@ -412,7 +413,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters(), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} val2 := []byte{0x1, 0x2, 0x3, 0x4} @@ -427,7 +428,7 @@ func TestStorageLimits(t *testing.T) { require.Equal(t, meter1.TotalBytesWrittenToStorage(), meter.GetStorageKeyValueSizeForTesting(key1, val2)) // first write of key2 - key2 := meter.StorageInteractionKey{Owner: "", Key: "2"} + key2 := flow.NewRegisterID("", "2") err = meter1.MeterStorageWrite(key2, val2, false) require.NoError(t, err) require.Equal(t, meter1.TotalBytesWrittenToStorage(), @@ -439,7 +440,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters().WithStorageInteractionLimit(1), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} err := meter1.MeterStorageRead(key1, val1, false /* not enforced */) @@ -453,7 +454,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters().WithStorageInteractionLimit(testLimit), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} err := meter1.MeterStorageRead(key1, val1, true /* enforced */) @@ -471,7 +472,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters().WithStorageInteractionLimit(testLimit), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} err := meter1.MeterStorageWrite(key1, val1, false /* not enforced */) @@ -484,7 +485,7 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters().WithStorageInteractionLimit(testLimit), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} + key1 := flow.NewRegisterID("", "1") val1 := []byte{0x1, 0x2, 0x3} err := meter1.MeterStorageWrite(key1, val1, true /* enforced */) @@ -501,8 +502,8 @@ func TestStorageLimits(t *testing.T) { meter.DefaultParameters(), ) - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} - key2 := meter.StorageInteractionKey{Owner: "", Key: "2"} + key1 := flow.NewRegisterID("", "1") + key2 := flow.NewRegisterID("", "2") val1 := []byte{0x1, 0x2, 0x3} val2 := []byte{0x1, 0x2, 0x3, 0x4} size1 := meter.GetStorageKeyValueSizeForTesting(key1, val1) @@ -522,8 +523,8 @@ func TestStorageLimits(t *testing.T) { }) t.Run("metering storage read and written - exceeding limit - not enforced", func(t *testing.T) { - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} - key2 := meter.StorageInteractionKey{Owner: "", Key: "2"} + key1 := flow.NewRegisterID("", "1") + key2 := flow.NewRegisterID("", "2") val1 := []byte{0x1, 0x2, 0x3} val2 := []byte{0x1, 0x2, 0x3, 0x4} size1 := meter.GetStorageKeyValueSizeForTesting(key1, val1) @@ -547,8 +548,8 @@ func TestStorageLimits(t *testing.T) { }) t.Run("metering storage read and written - exceeding limit - enforced", func(t *testing.T) { - key1 := meter.StorageInteractionKey{Owner: "", Key: "1"} - key2 := meter.StorageInteractionKey{Owner: "", Key: "2"} + key1 := flow.NewRegisterID("", "1") + key2 := flow.NewRegisterID("", "2") val1 := []byte{0x1, 0x2, 0x3} val2 := []byte{0x1, 0x2, 0x3, 0x4} size1 := meter.GetStorageKeyValueSizeForTesting(key1, val1) @@ -578,13 +579,13 @@ func TestStorageLimits(t *testing.T) { meter1 := meter.NewMeter( meter.DefaultParameters(), ) - readKey1 := meter.StorageInteractionKey{Owner: "", Key: "r1"} + readKey1 := flow.NewRegisterID("", "r1") readVal1 := []byte{0x1, 0x2, 0x3} readSize1 := meter.GetStorageKeyValueSizeForTesting(readKey1, readVal1) err := meter1.MeterStorageRead(readKey1, readVal1, false) require.NoError(t, err) - writeKey1 := meter.StorageInteractionKey{Owner: "", Key: "w1"} + writeKey1 := flow.NewRegisterID("", "w1") writeVal1 := []byte{0x1, 0x2, 0x3, 0x4} writeSize1 := meter.GetStorageKeyValueSizeForTesting(writeKey1, writeVal1) err = meter1.MeterStorageWrite(writeKey1, writeVal1, false) @@ -599,7 +600,7 @@ func TestStorageLimits(t *testing.T) { err = meter2.MeterStorageRead(readKey1, readVal1, false) require.NoError(t, err) - writeKey2 := meter.StorageInteractionKey{Owner: "", Key: "w2"} + writeKey2 := flow.NewRegisterID("", "w2") writeVal2 := []byte{0x1, 0x2, 0x3, 0x4, 0x5} writeSize2 := meter.GetStorageKeyValueSizeForTesting(writeKey2, writeVal2) err = meter2.MeterStorageWrite(writeKey2, writeVal2, false) diff --git a/fvm/programs/programs.go b/fvm/programs/programs.go index dd3eda696dd..a63a9bf8709 100644 --- a/fvm/programs/programs.go +++ b/fvm/programs/programs.go @@ -9,7 +9,7 @@ import ( "github.com/onflow/flow-go/fvm/state" ) -// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/229 +// TODO(patrick): rm after https://github.com/onflow/flow-emulator/pull/306 // is merged and integrated. type Programs struct { lock sync.RWMutex diff --git a/fvm/state/accounts_status.go b/fvm/state/accounts_status.go new file mode 100644 index 00000000000..9ee2442887c --- /dev/null +++ b/fvm/state/accounts_status.go @@ -0,0 +1,51 @@ +package state + +import ( + "encoding/hex" + + "github.com/onflow/flow-go/fvm/errors" +) + +type AccountStatus uint8 + +const ( + maskExist byte = 0b0000_0001 + maskFrozen byte = 0b1000_0000 +) + +// NewAccountStatus sets exist flag and return an AccountStatus +func NewAccountStatus() AccountStatus { + return AccountStatus(maskExist) +} + +func (a AccountStatus) ToBytes() []byte { + b := make([]byte, 1) + b[0] = byte(a) + return b +} + +func AccountStatusFromBytes(inp []byte) (AccountStatus, error) { + // if len of inp is zero, account does not exist + if len(inp) == 0 { + return 0, nil + } + if len(inp) > 1 { + return 0, errors.NewValueErrorf(hex.EncodeToString(inp), "invalid account state") + } + return AccountStatus(inp[0]), nil +} + +func (a AccountStatus) AccountExists() bool { + return a > 0 +} + +func (a AccountStatus) IsAccountFrozen() bool { + return uint8(a)&maskFrozen > 0 +} + +func SetAccountStatusFrozenFlag(inp AccountStatus, frozen bool) AccountStatus { + if frozen { + return AccountStatus(uint8(inp) | maskFrozen) + } + return AccountStatus(uint8(inp) & (0xFF - maskFrozen)) +} diff --git a/fvm/state/accounts_status_test.go b/fvm/state/accounts_status_test.go new file mode 100644 index 00000000000..2765fa15b72 --- /dev/null +++ b/fvm/state/accounts_status_test.go @@ -0,0 +1,30 @@ +package state_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/fvm/state" +) + +func TestAccountStatus(t *testing.T) { + + s := state.NewAccountStatus() + require.True(t, s.AccountExists()) + require.False(t, s.IsAccountFrozen()) + + s = state.SetAccountStatusFrozenFlag(s, true) + require.True(t, s.AccountExists()) + require.True(t, s.IsAccountFrozen()) + + s = state.SetAccountStatusFrozenFlag(s, false) + require.True(t, s.AccountExists()) + require.False(t, s.IsAccountFrozen()) + + var err error + s, err = state.AccountStatusFromBytes(s.ToBytes()) + require.NoError(t, err) + require.True(t, s.AccountExists()) + require.False(t, s.IsAccountFrozen()) +} diff --git a/fvm/state/state.go b/fvm/state/state.go index 860531f6f35..540c2ca5052 100644 --- a/fvm/state/state.go +++ b/fvm/state/state.go @@ -1,14 +1,9 @@ package state import ( - "bytes" - "encoding/binary" - "encoding/hex" "fmt" - "strings" "github.com/onflow/cadence/runtime/common" - "golang.org/x/exp/slices" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" @@ -18,19 +13,10 @@ import ( const ( DefaultMaxKeySize = 16_000 // ~16KB DefaultMaxValueSize = 256_000_000 // ~256MB - - // Service level keys (owner is empty): - UUIDKey = "uuid" - AddressStateKey = "account_address_state" - - // Account level keys - AccountKeyPrefix = "a." - AccountStatusKey = AccountKeyPrefix + "s" - CodeKeyPrefix = "code." - ContractNamesKey = "contract_names" - PublicKeyKeyPrefix = "public_key_" ) +// TODO(patrick): make State implement the View interface. +// // State represents the execution state // it holds draft of updates and captures // all register touches @@ -40,30 +26,25 @@ type State struct { // bookkeeping purpose). committed bool - view View - meter *meter.Meter - updatedAddresses map[flow.Address]struct{} - stateLimits -} + view View + meter *meter.Meter -type stateLimits struct { - maxKeySizeAllowed uint64 - maxValueSizeAllowed uint64 + // NOTE: parent and child state shares the same limits controller + *limitsController } type StateParameters struct { meter.MeterParameters - stateLimits + maxKeySizeAllowed uint64 + maxValueSizeAllowed uint64 } func DefaultParameters() StateParameters { return StateParameters{ - MeterParameters: meter.DefaultParameters(), - stateLimits: stateLimits{ - maxKeySizeAllowed: DefaultMaxKeySize, - maxValueSizeAllowed: DefaultMaxValueSize, - }, + MeterParameters: meter.DefaultParameters(), + maxKeySizeAllowed: DefaultMaxKeySize, + maxValueSizeAllowed: DefaultMaxValueSize, } } @@ -90,14 +71,28 @@ func (params StateParameters) WithMaxValueSizeAllowed(limit uint64) StateParamet return newParams } -// TODO(patrick): rm once https://github.com/onflow/flow-emulator/pull/245 -// is integrated. -// -// WithMaxInteractionSizeAllowed sets limit on total byte interaction with ledger -func (params StateParameters) WithMaxInteractionSizeAllowed(limit uint64) StateParameters { - newParams := params - newParams.MeterParameters = newParams.MeterParameters.WithStorageInteractionLimit(limit) - return newParams +type limitsController struct { + enforceLimits bool + maxKeySizeAllowed uint64 + maxValueSizeAllowed uint64 +} + +func newLimitsController(params StateParameters) *limitsController { + return &limitsController{ + enforceLimits: true, + maxKeySizeAllowed: params.maxKeySizeAllowed, + maxValueSizeAllowed: params.maxValueSizeAllowed, + } +} + +func (controller *limitsController) RunWithAllLimitsDisabled(f func()) { + if f == nil { + return + } + current := controller.enforceLimits + controller.enforceLimits = false + f() + controller.enforceLimits = current } func (s *State) View() View { @@ -117,8 +112,7 @@ func NewState(view View, params StateParameters) *State { committed: false, view: view, meter: m, - updatedAddresses: make(map[flow.Address]struct{}), - stateLimits: params.stateLimits, + limitsController: newLimitsController(params), } } @@ -131,8 +125,7 @@ func (s *State) NewChildWithMeterParams( committed: false, view: s.view.NewChild(), meter: meter.NewMeter(params), - updatedAddresses: make(map[flow.Address]struct{}), - stateLimits: s.stateLimits, + limitsController: s.limitsController, } } @@ -146,13 +139,23 @@ func (s *State) InteractionUsed() uint64 { return s.meter.TotalBytesOfStorageInteractions() } -// UpdatedRegisters returns the lists of register id / value that were updated. +// BytesWritten returns the amount of total ledger bytes written +func (s *State) BytesWritten() uint64 { + return s.meter.TotalBytesWrittenToStorage() +} + +// UpdatedRegisterIDs returns the lists of register ids that were updated. +func (s *State) UpdatedRegisterIDs() []flow.RegisterID { + return s.view.UpdatedRegisterIDs() +} + +// UpdatedRegisters returns the lists of register entries that were updated. func (s *State) UpdatedRegisters() flow.RegisterEntries { return s.view.UpdatedRegisters() } // Get returns a register value given owner and key -func (s *State) Get(owner, key string, enforceLimit bool) (flow.RegisterValue, error) { +func (s *State) Get(id flow.RegisterID) (flow.RegisterValue, error) { if s.committed { return nil, fmt.Errorf("cannot Get on a committed state") } @@ -160,60 +163,43 @@ func (s *State) Get(owner, key string, enforceLimit bool) (flow.RegisterValue, e var value []byte var err error - if enforceLimit { - if err = s.checkSize(owner, key, []byte{}); err != nil { + if s.enforceLimits { + if err = s.checkSize(id, []byte{}); err != nil { return nil, err } } - if value, err = s.view.Get(owner, key); err != nil { + if value, err = s.view.Get(id); err != nil { // wrap error into a fatal error getError := errors.NewLedgerFailure(err) // wrap with more info - return nil, fmt.Errorf("failed to read key %s on account %s: %w", PrintableKey(key), hex.EncodeToString([]byte(owner)), getError) + return nil, fmt.Errorf("failed to read %s: %w", id, getError) } - err = s.meter.MeterStorageRead( - meter.StorageInteractionKey{Owner: owner, Key: key}, - value, - enforceLimit) - + err = s.meter.MeterStorageRead(id, value, s.enforceLimits) return value, err } // Set updates state delta with a register update -func (s *State) Set(owner, key string, value flow.RegisterValue, enforceLimit bool) error { +func (s *State) Set(id flow.RegisterID, value flow.RegisterValue) error { if s.committed { return fmt.Errorf("cannot Set on a committed state") } - if enforceLimit { - if err := s.checkSize(owner, key, value); err != nil { + if s.enforceLimits { + if err := s.checkSize(id, value); err != nil { return err } } - if err := s.view.Set(owner, key, value); err != nil { + if err := s.view.Set(id, value); err != nil { // wrap error into a fatal error setError := errors.NewLedgerFailure(err) // wrap with more info - return fmt.Errorf("failed to update key %s on account %s: %w", PrintableKey(key), hex.EncodeToString([]byte(owner)), setError) + return fmt.Errorf("failed to update %s: %w", id, setError) } - err := s.meter.MeterStorageWrite( - meter.StorageInteractionKey{Owner: owner, Key: key}, - value, - enforceLimit, - ) - if err != nil { - return err - } - - if address, isAddress := addressFromOwner(owner); isAddress { - s.updatedAddresses[address] = struct{}{} - } - - return nil + return s.meter.MeterStorageWrite(id, value, s.enforceLimits) } // MeterComputation meters computation usage @@ -222,7 +208,10 @@ func (s *State) MeterComputation(kind common.ComputationKind, intensity uint) er return fmt.Errorf("cannot MeterComputation on a committed state") } - return s.meter.MeterComputation(kind, intensity) + if s.enforceLimits { + return s.meter.MeterComputation(kind, intensity) + } + return nil } // TotalComputationUsed returns total computation used @@ -246,7 +235,11 @@ func (s *State) MeterMemory(kind common.MemoryKind, intensity uint) error { return fmt.Errorf("cannot MeterMemory on a committed state") } - return s.meter.MeterMemory(kind, intensity) + if s.enforceLimits { + return s.meter.MeterMemory(kind, intensity) + } + + return nil } // MemoryIntensities returns computation intensities @@ -269,7 +262,11 @@ func (s *State) MeterEmittedEvent(byteSize uint64) error { return fmt.Errorf("cannot MeterEmittedEvent on a committed state") } - return s.meter.MeterEmittedEvent(byteSize) + if s.enforceLimits { + return s.meter.MeterEmittedEvent(byteSize) + } + + return nil } func (s *State) TotalEmittedEventBytes() uint64 { @@ -289,91 +286,26 @@ func (s *State) MergeState(other *State) error { s.meter.MergeMeter(other.meter) - // apply address updates - for k, v := range other.updatedAddresses { - s.updatedAddresses[k] = v - } - return nil } -// UpdatedAddresses returns a sorted list of addresses that were updated (at least 1 register update) -func (s *State) UpdatedAddresses() []flow.Address { - addresses := make([]flow.Address, 0, len(s.updatedAddresses)) - - for k := range s.updatedAddresses { - addresses = append(addresses, k) - } - - slices.SortFunc(addresses, func(a, b flow.Address) bool { - // reverse order to maintain compatibility with previous implementation. - return bytes.Compare(a[:], b[:]) >= 0 - }) - - return addresses -} - -func (s *State) checkSize(owner, key string, value flow.RegisterValue) error { - keySize := uint64(len(owner) + len(key)) +func (s *State) checkSize( + id flow.RegisterID, + value flow.RegisterValue, +) error { + keySize := uint64(len(id.Owner) + len(id.Key)) valueSize := uint64(len(value)) if keySize > s.maxKeySizeAllowed { - return errors.NewStateKeySizeLimitError(owner, key, keySize, s.maxKeySizeAllowed) + return errors.NewStateKeySizeLimitError( + id, + keySize, + s.maxKeySizeAllowed) } if valueSize > s.maxValueSizeAllowed { - return errors.NewStateValueSizeLimitError(value, valueSize, s.maxValueSizeAllowed) + return errors.NewStateValueSizeLimitError( + value, + valueSize, + s.maxValueSizeAllowed) } return nil } - -func addressFromOwner(owner string) (flow.Address, bool) { - ownerBytes := []byte(owner) - if len(ownerBytes) != flow.AddressLength { - // not an address - return flow.EmptyAddress, false - } - address := flow.BytesToAddress(ownerBytes) - return address, true -} - -// IsFVMStateKey returns true if the key is controlled by the fvm env and -// return false otherwise (key controlled by the cadence env) -func IsFVMStateKey(owner string, key string) bool { - // check if is a service level key (owner is empty) - // cases: - // - "", "uuid" - // - "", "account_address_state" - if len(owner) == 0 && (key == UUIDKey || key == AddressStateKey) { - return true - } - - // check account level keys - // cases: - // - address, "contract_names" - // - address, "code.%s" (contract name) - // - address, "public_key_%d" (index) - // - address, "a.s" (account status) - return strings.HasPrefix(key, PublicKeyKeyPrefix) || - key == ContractNamesKey || - strings.HasPrefix(key, CodeKeyPrefix) || - key == AccountStatusKey -} - -// This returns true if the key is a slab index for an account's ordered fields -// map. -// -// In general, each account's regular fields are stored in ordered map known -// only to cadence. Cadence encodes this map into bytes and split the bytes -// into slab chunks before storing the slabs into the ledger. -func IsSlabIndex(key string) bool { - return len(key) == 9 && key[0] == '$' -} - -// PrintableKey formats slabs properly and avoids invalid utf8s -func PrintableKey(key string) string { - // slab - if IsSlabIndex(key) { - i := uint64(binary.BigEndian.Uint64([]byte(key[1:]))) - return fmt.Sprintf("$%d", i) - } - return fmt.Sprintf("#%x", []byte(key)) -} diff --git a/fvm/state/state_test.go b/fvm/state/state_test.go index 42e49a524bd..85def48f133 100644 --- a/fvm/state/state_test.go +++ b/fvm/state/state_test.go @@ -2,15 +2,13 @@ package state_test import ( "testing" - "unicode/utf8" "github.com/stretchr/testify/require" - "github.com/onflow/atree" - "github.com/onflow/flow-go/fvm/meter" "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/utils" + "github.com/onflow/flow-go/model/flow" ) func createByteArray(size int) []byte { @@ -26,45 +24,45 @@ func TestState_ChildMergeFunctionality(t *testing.T) { st := state.NewState(view, state.DefaultParameters()) t.Run("test read from parent state (backoff)", func(t *testing.T) { - key := "key1" + key := flow.NewRegisterID("address", "key1") value := createByteArray(1) // set key1 on parent - err := st.Set("address", key, value, true) + err := st.Set(key, value) require.NoError(t, err) // read key1 on child stChild := st.NewChild() - v, err := stChild.Get("address", key, true) + v, err := stChild.Get(key) require.NoError(t, err) require.Equal(t, v, value) }) t.Run("test write to child (no merge)", func(t *testing.T) { - key := "key2" + key := flow.NewRegisterID("address", "key2") value := createByteArray(2) stChild := st.NewChild() // set key2 on child - err := stChild.Set("address", key, value, true) + err := stChild.Set(key, value) require.NoError(t, err) // read key2 on parent - v, err := st.Get("address", key, true) + v, err := st.Get(key) require.NoError(t, err) require.Equal(t, len(v), 0) }) t.Run("test write to child and merge", func(t *testing.T) { - key := "key3" + key := flow.NewRegisterID("address", "key3") value := createByteArray(3) stChild := st.NewChild() // set key3 on child - err := stChild.Set("address", key, value, true) + err := stChild.Set(key, value) require.NoError(t, err) // read before merge - v, err := st.Get("address", key, true) + v, err := st.Get(key) require.NoError(t, err) require.Equal(t, len(v), 0) @@ -73,20 +71,20 @@ func TestState_ChildMergeFunctionality(t *testing.T) { require.NoError(t, err) // read key3 on parent - v, err = st.Get("address", key, true) + v, err = st.Get(key) require.NoError(t, err) require.Equal(t, v, value) }) t.Run("test write to ledger", func(t *testing.T) { - key := "key4" + key := flow.NewRegisterID("address", "key4") value := createByteArray(4) // set key4 on parent - err := st.Set("address", key, value, true) + err := st.Set(key, value) require.NoError(t, err) // now should be part of the ledger - v, err := view.Get("address", key) + v, err := view.Get(key) require.NoError(t, err) require.Equal(t, v, value) }) @@ -97,121 +95,120 @@ func TestState_MaxValueSize(t *testing.T) { view := utils.NewSimpleView() st := state.NewState(view, state.DefaultParameters().WithMaxValueSizeAllowed(6)) + key := flow.NewRegisterID("address", "key") + // update should pass value := createByteArray(5) - err := st.Set("address", "key", value, true) + err := st.Set(key, value) require.NoError(t, err) // update shouldn't pass value = createByteArray(7) - err = st.Set("address", "key", value, true) + err = st.Set(key, value) require.Error(t, err) } func TestState_MaxKeySize(t *testing.T) { view := utils.NewSimpleView() - st := state.NewState(view, state.DefaultParameters().WithMaxKeySizeAllowed(4)) + st := state.NewState( + view, + // Note: owners are always 8 bytes + state.DefaultParameters().WithMaxKeySizeAllowed(8+2)) + + key1 := flow.NewRegisterID("1", "23") + key2 := flow.NewRegisterID("123", "234") // read - _, err := st.Get("1", "2", true) + _, err := st.Get(key1) require.NoError(t, err) // read - _, err = st.Get("123", "234", true) + _, err = st.Get(key2) require.Error(t, err) // update - err = st.Set("1", "2", []byte{}, true) + err = st.Set(key1, []byte{}) require.NoError(t, err) // read - err = st.Set("123", "234", []byte{}, true) + err = st.Set(key2, []byte{}) require.Error(t, err) } func TestState_MaxInteraction(t *testing.T) { view := utils.NewSimpleView() + + key1 := flow.NewRegisterID("1", "2") + key1Size := uint64(8 + 1) + + value1 := []byte("A") + value1Size := uint64(1) + + key2 := flow.NewRegisterID("123", "23") + key2Size := uint64(8 + 2) + + key3 := flow.NewRegisterID("234", "345") + key3Size := uint64(8 + 3) + + key4 := flow.NewRegisterID("3", "4567") + key4Size := uint64(8 + 4) + st := state.NewState( view, state.DefaultParameters(). WithMeterParameters( - meter.DefaultParameters().WithStorageInteractionLimit(12))) + meter.DefaultParameters().WithStorageInteractionLimit( + key1Size+key2Size+key3Size-1))) // read - interaction 2 - _, err := st.Get("1", "2", true) - require.Equal(t, st.InteractionUsed(), uint64(2)) + _, err := st.Get(key1) + require.Equal(t, st.InteractionUsed(), key1Size) require.NoError(t, err) // read - interaction 8 - _, err = st.Get("123", "234", true) - require.Equal(t, st.InteractionUsed(), uint64(8)) + _, err = st.Get(key2) require.NoError(t, err) + require.Equal(t, st.InteractionUsed(), key1Size+key2Size) // read - interaction 14 - _, err = st.Get("234", "345", true) - require.Equal(t, st.InteractionUsed(), uint64(14)) + _, err = st.Get(key3) require.Error(t, err) + require.Equal(t, st.InteractionUsed(), key1Size+key2Size+key3Size) st = state.NewState( view, state.DefaultParameters(). WithMeterParameters( - meter.DefaultParameters().WithStorageInteractionLimit(6))) + meter.DefaultParameters().WithStorageInteractionLimit( + key1Size+value1Size+key2Size))) stChild := st.NewChild() // update - 0 - err = stChild.Set("1", "2", []byte{'A'}, true) + err = stChild.Set(key1, value1) require.NoError(t, err) require.Equal(t, st.InteractionUsed(), uint64(0)) // commit err = st.MergeState(stChild) require.NoError(t, err) - require.Equal(t, st.InteractionUsed(), uint64(3)) + require.Equal(t, st.InteractionUsed(), key1Size+value1Size) // read - interaction 3 (already in read cache) - _, err = st.Get("1", "2", true) + _, err = st.Get(key1) require.NoError(t, err) - require.Equal(t, st.InteractionUsed(), uint64(3)) + require.Equal(t, st.InteractionUsed(), key1Size+value1Size) // read - interaction 5 - _, err = st.Get("2", "3", true) + _, err = st.Get(key2) require.NoError(t, err) - require.Equal(t, st.InteractionUsed(), uint64(5)) + require.Equal(t, st.InteractionUsed(), key1Size+value1Size+key2Size) // read - interaction 7 - _, err = st.Get("3", "4", true) + _, err = st.Get(key4) require.Error(t, err) - -} - -func TestState_IsFVMStateKey(t *testing.T) { - require.True(t, state.IsFVMStateKey("", state.UUIDKey)) - require.True(t, state.IsFVMStateKey("", state.AddressStateKey)) - require.False(t, state.IsFVMStateKey("", "other")) - require.False(t, state.IsFVMStateKey("Address", state.UUIDKey)) - require.False(t, state.IsFVMStateKey("Address", state.AddressStateKey)) - require.True(t, state.IsFVMStateKey("Address", "public_key_12")) - require.True(t, state.IsFVMStateKey("Address", state.ContractNamesKey)) - require.True(t, state.IsFVMStateKey("Address", "code.MYCODE")) - require.True(t, state.IsFVMStateKey("Address", state.AccountStatusKey)) - require.False(t, state.IsFVMStateKey("Address", "anything else")) -} - -func TestAccounts_PrintableKey(t *testing.T) { - // slab with 189 should result in \\xbd - slabIndex := atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 189}) - key := string(atree.SlabIndexToLedgerKey(slabIndex)) - require.False(t, utf8.ValidString(key)) - printable := state.PrintableKey(key) - require.True(t, utf8.ValidString(printable)) - require.Equal(t, "$189", printable) - - // non slab invalid utf-8 - key = "a\xc5z" - require.False(t, utf8.ValidString(key)) - printable = state.PrintableKey(key) - require.True(t, utf8.ValidString(printable)) - require.Equal(t, "#61c57a", printable) + require.Equal( + t, + st.InteractionUsed(), + key1Size+value1Size+key2Size+key4Size) } diff --git a/fvm/state/transaction_state.go b/fvm/state/transaction_state.go index f71e0049269..a994e53e64a 100644 --- a/fvm/state/transaction_state.go +++ b/fvm/state/transaction_state.go @@ -23,8 +23,6 @@ type nestedTransactionStackFrame struct { // TransactionState provides active transaction states and facilitates common // state management operations. type TransactionState struct { - enforceLimits bool - // NOTE: The first frame is always the main transaction, and is not // poppable during the course of the transaction. nestedTransactions []nestedTransactionStackFrame @@ -47,7 +45,6 @@ func NewTransactionState( ) *TransactionState { startState := NewState(startView, params) return &TransactionState{ - enforceLimits: true, nestedTransactions: []nestedTransactionStackFrame{ nestedTransactionStackFrame{ state: startState, @@ -331,27 +328,19 @@ func (s *TransactionState) RestartNestedTransaction( } func (s *TransactionState) Get( - owner string, - key string, - enforceLimit bool, + id flow.RegisterID, ) ( flow.RegisterValue, error, ) { - return s.currentState().Get(owner, key, enforceLimit) + return s.currentState().Get(id) } func (s *TransactionState) Set( - owner string, - key string, + id flow.RegisterID, value flow.RegisterValue, - enforceLimit bool, ) error { - return s.currentState().Set(owner, key, value, enforceLimit) -} - -func (s *TransactionState) UpdatedAddresses() []flow.Address { - return s.currentState().UpdatedAddresses() + return s.currentState().Set(id, value) } func (s *TransactionState) MeterComputation( @@ -404,38 +393,15 @@ func (s *TransactionState) ViewForTestingOnly() View { return s.currentState().View() } -func (s *TransactionState) UpdatedRegisters() flow.RegisterEntries { - return s.currentState().UpdatedRegisters() -} - -// EnableAllLimitEnforcements enables all the limits -func (s *TransactionState) EnableAllLimitEnforcements() { - s.enforceLimits = true +func (s *TransactionState) UpdatedRegisterIDs() []flow.RegisterID { + return s.currentState().UpdatedRegisterIDs() } -// DisableAllLimitEnforcements disables all the limits -func (s *TransactionState) DisableAllLimitEnforcements() { - s.enforceLimits = false +func (s *TransactionState) UpdatedRegisters() flow.RegisterEntries { + return s.currentState().UpdatedRegisters() } // RunWithAllLimitsDisabled runs f with limits disabled func (s *TransactionState) RunWithAllLimitsDisabled(f func()) { - if f == nil { - return - } - current := s.enforceLimits - s.enforceLimits = false - f() - s.enforceLimits = current -} - -// EnforceComputationLimits returns if the computation limits should be enforced -// or not. -func (s *TransactionState) EnforceComputationLimits() bool { - return s.enforceLimits -} - -// EnforceInteractionLimits returns if the interaction limits should be enforced or not -func (s *TransactionState) EnforceLimits() bool { - return s.enforceLimits + s.currentState().RunWithAllLimitsDisabled(f) } diff --git a/fvm/state/transaction_state_test.go b/fvm/state/transaction_state_test.go index ed38c892fb8..ba2c0760df6 100644 --- a/fvm/state/transaction_state_test.go +++ b/fvm/state/transaction_state_test.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/fvm/meter" "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/utils" + "github.com/onflow/flow-go/model/flow" ) func newTestTransactionState() *state.TransactionState { @@ -50,22 +51,21 @@ func TestUnrestrictedNestedTransactionBasic(t *testing.T) { // Ensure the values are written to the correctly nested state - addr := "address" - key := "key" + key := flow.NewRegisterID("address", "key") val := createByteArray(2) - err = txn.Set(addr, key, val, true) + err = txn.Set(key, val) require.NoError(t, err) - v, err := nestedState2.Get(addr, key, true) + v, err := nestedState2.Get(key) require.NoError(t, err) require.Equal(t, val, v) - v, err = nestedState1.Get(addr, key, true) + v, err = nestedState1.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -77,11 +77,11 @@ func TestUnrestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 1, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(id1)) - v, err = nestedState1.Get(addr, key, true) + v, err = nestedState1.Get(key) require.NoError(t, err) require.Equal(t, val, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -91,7 +91,7 @@ func TestUnrestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 0, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(txn.MainTransactionId())) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Equal(t, val, v) } @@ -174,22 +174,21 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { // Sanity check - addr := "address" - key := "key" + key := flow.NewRegisterID("address", "key") - v, err := restrictedNestedState2.Get(addr, key, true) + v, err := restrictedNestedState2.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = restrictedNestedState1.Get(addr, key, true) + v, err = restrictedNestedState1.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = nestedState.Get(addr, key, true) + v, err = nestedState.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -202,7 +201,7 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { state.DefaultParameters(), ) - err = cachedState.Set(addr, key, val, true) + err = cachedState.Set(key, val) require.NoError(t, err) err = txn.AttachAndCommit(cachedState) @@ -211,19 +210,19 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 3, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(restrictedId2)) - v, err = restrictedNestedState2.Get(addr, key, true) + v, err = restrictedNestedState2.Get(key) require.NoError(t, err) require.Equal(t, val, v) - v, err = restrictedNestedState1.Get(addr, key, true) + v, err = restrictedNestedState1.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = nestedState.Get(addr, key, true) + v, err = nestedState.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -236,15 +235,15 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 2, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(restrictedId1)) - v, err = restrictedNestedState1.Get(addr, key, true) + v, err = restrictedNestedState1.Get(key) require.NoError(t, err) require.Equal(t, val, v) - v, err = nestedState.Get(addr, key, true) + v, err = nestedState.Get(key) require.NoError(t, err) require.Nil(t, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -255,11 +254,11 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 1, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(id1)) - v, err = nestedState.Get(addr, key, true) + v, err = nestedState.Get(key) require.NoError(t, err) require.Equal(t, val, v) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Nil(t, v) @@ -269,7 +268,7 @@ func TestParseRestrictedNestedTransactionBasic(t *testing.T) { require.Equal(t, 0, txn.NumNestedTransactions()) require.True(t, txn.IsCurrent(mainId)) - v, err = mainState.Get(addr, key, true) + v, err = mainState.Get(key) require.NoError(t, err) require.Equal(t, val, v) } @@ -282,15 +281,14 @@ func TestRestartNestedTransaction(t *testing.T) { id, err := txn.BeginNestedTransaction() require.NoError(t, err) - addr := "address" - key := "key" + key := flow.NewRegisterID("address", "key") val := createByteArray(2) for i := 0; i < 10; i++ { _, err := txn.BeginNestedTransaction() require.NoError(t, err) - err = txn.Set(addr, key, val, true) + err = txn.Set(key, val) require.NoError(t, err) } @@ -303,7 +301,7 @@ func TestRestartNestedTransaction(t *testing.T) { _, err := txn.BeginParseRestrictedNestedTransaction(loc) require.NoError(t, err) - err = txn.Set(addr, key, val, true) + err = txn.Set(key, val) require.NoError(t, err) } @@ -322,7 +320,7 @@ func TestRestartNestedTransaction(t *testing.T) { require.Greater(t, state.InteractionUsed(), uint64(0)) - v, err := state.Get(addr, key, true) + v, err := state.Get(key) require.NoError(t, err) require.Nil(t, v) } @@ -335,11 +333,10 @@ func TestRestartNestedTransactionWithInvalidId(t *testing.T) { id, err := txn.BeginNestedTransaction() require.NoError(t, err) - addr := "address" - key := "key" + key := flow.NewRegisterID("address", "key") val := createByteArray(2) - err = txn.Set(addr, key, val, true) + err = txn.Set(key, val) require.NoError(t, err) var otherId state.NestedTransactionId @@ -358,7 +355,7 @@ func TestRestartNestedTransactionWithInvalidId(t *testing.T) { require.True(t, txn.IsCurrent(id)) - v, err := txn.Get(addr, key, true) + v, err := txn.Get(key) require.NoError(t, err) require.Equal(t, val, v) } @@ -486,40 +483,43 @@ func TestParseRestrictedCannotCommitLocationMismatch(t *testing.T) { func TestPauseAndResume(t *testing.T) { txn := newTestTransactionState() - val, err := txn.Get("addr", "key", true) + key1 := flow.NewRegisterID("addr", "key") + key2 := flow.NewRegisterID("addr2", "key2") + + val, err := txn.Get(key1) require.NoError(t, err) require.Nil(t, val) id1, err := txn.BeginNestedTransaction() require.NoError(t, err) - err = txn.Set("addr", "key", createByteArray(2), true) + err = txn.Set(key1, createByteArray(2)) require.NoError(t, err) - val, err = txn.Get("addr", "key", true) + val, err = txn.Get(key1) require.NoError(t, err) require.NotNil(t, val) pausedState, err := txn.Pause(id1) require.NoError(t, err) - val, err = txn.Get("addr", "key", true) + val, err = txn.Get(key1) require.NoError(t, err) require.Nil(t, val) txn.Resume(pausedState) - val, err = txn.Get("addr", "key", true) + val, err = txn.Get(key1) require.NoError(t, err) require.NotNil(t, val) - err = txn.Set("addr2", "key2", createByteArray(2), true) + err = txn.Set(key2, createByteArray(2)) require.NoError(t, err) _, err = txn.Commit(id1) require.NoError(t, err) - val, err = txn.Get("addr2", "key2", true) + val, err = txn.Get(key2) require.NoError(t, err) require.NotNil(t, val) } @@ -530,10 +530,11 @@ func TestInvalidCommittedStateModification(t *testing.T) { id1, err := txn.BeginNestedTransaction() require.NoError(t, err) - err = txn.Set("addr", "key", createByteArray(2), true) + key := flow.NewRegisterID("addr", "key") + err = txn.Set(key, createByteArray(2)) require.NoError(t, err) - _, err = txn.Get("addr", "key", true) + _, err = txn.Get(key) require.NoError(t, err) committedState, err := txn.Commit(id1) @@ -545,10 +546,10 @@ func TestInvalidCommittedStateModification(t *testing.T) { txn.Resume(committedState) - err = txn.Set("addr", "key", createByteArray(2), true) + err = txn.Set(key, createByteArray(2)) require.ErrorContains(t, err, "cannot Set on a committed state") - _, err = txn.Get("addr", "key", true) + _, err = txn.Get(key) require.ErrorContains(t, err, "cannot Get on a committed state") _, err = txn.Commit(id1) diff --git a/fvm/state/view.go b/fvm/state/view.go index e3328b7692a..2480407b9af 100644 --- a/fvm/state/view.go +++ b/fvm/state/view.go @@ -8,18 +8,26 @@ type View interface { NewChild() View MergeView(child View) error DropDelta() // drops all the delta changes + + // UpdatedRegisters returns all registers that were updated by this view. + // The returned entries are sorted by ids. UpdatedRegisters() flow.RegisterEntries - AllRegisters() []flow.RegisterID + + // UpdatedRegisterIDs returns all register ids that were updated by this + // view. The returned ids are unsorted. + UpdatedRegisterIDs() []flow.RegisterID + + // AllRegisterIDs returns all register ids that were touched by this view. + // The returned ids are unsorted. + AllRegisterIDs() []flow.RegisterID + Ledger } // Ledger is the storage interface used by the virtual machine to read and write register values. // // TODO Rename this to Storage -// and remove reference to flow.RegisterValue and use byte[] type Ledger interface { - Set(owner, key string, value flow.RegisterValue) error - Get(owner, key string) (flow.RegisterValue, error) - Touch(owner, key string) error - Delete(owner, key string) error + Set(id flow.RegisterID, value flow.RegisterValue) error + Get(id flow.RegisterID) (flow.RegisterValue, error) } diff --git a/fvm/transaction.go b/fvm/transaction.go index f0726c7e601..83bd412919a 100644 --- a/fvm/transaction.go +++ b/fvm/transaction.go @@ -38,8 +38,9 @@ type TransactionProcedure struct { TxIndex uint32 Logs []string - Events []flow.Event - ServiceEvents []flow.Event + Events flow.EventsList + ServiceEvents flow.EventsList + ConvertedServiceEvents flow.ServiceEventList ComputationUsed uint64 ComputationIntensities meter.MeteredComputationIntensities MemoryEstimate uint64 diff --git a/fvm/transactionInvoker.go b/fvm/transactionInvoker.go index f26e8c88336..f6f55311d82 100644 --- a/fvm/transactionInvoker.go +++ b/fvm/transactionInvoker.go @@ -278,7 +278,7 @@ func (executor *transactionExecutor) ExecuteTransactionBody() error { if executor.errs.CollectedError() { invalidator = nil - executor.errorExecution() + executor.txnState.RunWithAllLimitsDisabled(executor.errorExecution) if executor.errs.CollectedFailure() { return executor.errs.ErrorOrNil() } @@ -387,7 +387,7 @@ func (executor *transactionExecutor) normalExecution() ( executor.txnState.RunWithAllLimitsDisabled(func() { err = executor.CheckStorageLimits( executor.env, - executor.txnState.UpdatedAddresses(), + executor.txnState, executor.proc.Transaction.Payer, maxTxFees) }) @@ -405,9 +405,6 @@ func (executor *transactionExecutor) normalExecution() ( // Clear changes and try to deduct fees again. func (executor *transactionExecutor) errorExecution() { - executor.txnState.DisableAllLimitEnforcements() - defer executor.txnState.EnableAllLimitEnforcements() - // log transaction as failed executor.env.Logger().Info(). Err(executor.errs.ErrorOrNil()). @@ -460,6 +457,7 @@ func (executor *transactionExecutor) commit( // if tx failed this will only contain fee deduction events executor.proc.Events = executor.env.Events() executor.proc.ServiceEvents = executor.env.ServiceEvents() + executor.proc.ConvertedServiceEvents = executor.env.ConvertedServiceEvents() // Based on various (e.g., contract and frozen account) updates, we decide // how to clean up the derived data. For failed transactions we also do diff --git a/fvm/transactionStorageLimiter.go b/fvm/transactionStorageLimiter.go index 1f44d6bde84..d05b7b16e2b 100644 --- a/fvm/transactionStorageLimiter.go +++ b/fvm/transactionStorageLimiter.go @@ -1,17 +1,28 @@ package fvm import ( + "bytes" "fmt" "github.com/onflow/cadence" "github.com/onflow/cadence/runtime/common" + "golang.org/x/exp/slices" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" ) +func addressFromRegisterId(id flow.RegisterID) (flow.Address, bool) { + if len(id.Owner) != flow.AddressLength { + return flow.EmptyAddress, false + } + + return flow.BytesToAddress([]byte(id.Owner)), true +} + type TransactionStorageLimiter struct{} // CheckStorageLimits checks each account that had its storage written to during a transaction, that its storage used @@ -21,9 +32,9 @@ type TransactionStorageLimiter struct{} // // The payers balance is considered to be maxTxFees lower that its actual balance, due to the fact that // the fee deduction step happens after the storage limit check. -func (d TransactionStorageLimiter) CheckStorageLimits( +func (limiter TransactionStorageLimiter) CheckStorageLimits( env environment.Environment, - addresses []flow.Address, + txnState *state.TransactionState, payer flow.Address, maxTxFees uint64, ) error { @@ -33,35 +44,70 @@ func (d TransactionStorageLimiter) CheckStorageLimits( defer env.StartChildSpan(trace.FVMTransactionStorageUsedCheck).End() - err := d.checkStorageLimits(env, addresses, payer, maxTxFees) + err := limiter.checkStorageLimits(env, txnState, payer, maxTxFees) if err != nil { return fmt.Errorf("storage limit check failed: %w", err) } return nil } -func (d TransactionStorageLimiter) checkStorageLimits( - env environment.Environment, - addresses []flow.Address, +// getStorageCheckAddresses returns a list of addresses to be checked whether +// storage limit is exceeded. The returned list include addresses of updated +// registers (and the payer's address). +func (limiter TransactionStorageLimiter) getStorageCheckAddresses( + txnState *state.TransactionState, payer flow.Address, maxTxFees uint64, -) error { - // in case the payer is not already part of the check, include it here. - // If the maxTxFees is zero, it doesn't matter if the payer is included or not. +) []flow.Address { + updatedIds := txnState.UpdatedRegisterIDs() + + // Multiple updated registers might be from the same address. We want to + // duplicated the addresses to reduce check overhead. + dedup := make(map[flow.Address]struct{}, len(updatedIds)+1) + addresses := make([]flow.Address, 0, len(updatedIds)+1) + + // In case the payer is not updated, include it here. If the maxTxFees is + // zero, it doesn't matter if the payer is included or not. if maxTxFees > 0 { - commonAddressesContainPayer := false - for _, address := range addresses { - if address == payer { - commonAddressesContainPayer = true - break - } + dedup[payer] = struct{}{} + addresses = append(addresses, payer) + } + + for _, id := range updatedIds { + address, ok := addressFromRegisterId(id) + if !ok { + continue } - if !commonAddressesContainPayer { - addresses = append(addresses, payer) + _, ok = dedup[address] + if ok { + continue } + + dedup[address] = struct{}{} + addresses = append(addresses, address) } + slices.SortFunc( + addresses, + func(a flow.Address, b flow.Address) bool { + // reverse order to maintain compatibility with previous + // implementation. + return bytes.Compare(a[:], b[:]) >= 0 + }) + return addresses +} + +// checkStorageLimits checks if the transaction changed the storage of any +// address and exceeded the storage limit. +func (limiter TransactionStorageLimiter) checkStorageLimits( + env environment.Environment, + txnState *state.TransactionState, + payer flow.Address, + maxTxFees uint64, +) error { + addresses := limiter.getStorageCheckAddresses(txnState, payer, maxTxFees) + commonAddresses := make([]common.Address, len(addresses)) usages := make([]uint64, len(commonAddresses)) @@ -82,23 +128,32 @@ func (d TransactionStorageLimiter) checkStorageLimits( maxTxFees, ) - // This error only occurs in case of implementation errors. The InvokeAccountsStorageCapacity - // already handles cases where the default vault is missing. + // This error only occurs in case of implementation errors. + // InvokeAccountsStorageCapacity already handles cases where the default + // vault is missing. if invokeErr != nil { return invokeErr } - // the resultArray elements are in the same order as the addresses and the addresses are deterministically sorted + // The resultArray elements are in the same order as the addresses and the + // addresses are deterministically sorted. resultArray, ok := result.(cadence.Array) if !ok { return fmt.Errorf("AccountsStorageCapacity did not return an array") } - for i, value := range resultArray.Values { - capacity := environment.StorageMBUFixToBytesUInt(value) + if len(addresses) != len(resultArray.Values) { + return fmt.Errorf("number of addresses does not match number of result") + } + + for i, address := range addresses { + capacity := environment.StorageMBUFixToBytesUInt(resultArray.Values[i]) if usages[i] > capacity { - return errors.NewStorageCapacityExceededError(flow.BytesToAddress(addresses[i].Bytes()), usages[i], capacity) + return errors.NewStorageCapacityExceededError( + address, + usages[i], + capacity) } } diff --git a/fvm/transactionStorageLimiter_test.go b/fvm/transactionStorageLimiter_test.go index 8f761f2ee76..60f646acc26 100644 --- a/fvm/transactionStorageLimiter_test.go +++ b/fvm/transactionStorageLimiter_test.go @@ -10,12 +10,28 @@ import ( "github.com/onflow/flow-go/fvm" fvmmock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/errors" + "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/fvm/utils" "github.com/onflow/flow-go/model/flow" ) func TestTransactionStorageLimiter(t *testing.T) { + txnState := state.NewTransactionState( + utils.NewSimpleView(), + state.DefaultParameters()) + owner := flow.HexToAddress("1") + + err := txnState.Set( + flow.NewRegisterID(string(owner[:]), "a"), + flow.RegisterValue("foo")) + require.NoError(t, err) + err = txnState.Set( + flow.NewRegisterID(string(owner[:]), "b"), + flow.RegisterValue("bar")) + require.NoError(t, err) + t.Run("capacity > storage -> OK", func(t *testing.T) { chain := flow.Mainnet.Chain() env := &fvmmock.Environment{} @@ -32,7 +48,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, []flow.Address{owner}, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, txnState, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("capacity = storage -> OK", func(t *testing.T) { @@ -51,7 +67,26 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, []flow.Address{owner}, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, txnState, flow.EmptyAddress, 0) + require.NoError(t, err, "Transaction with equal capacity than storage used should work") + }) + t.Run("capacity = storage -> OK (dedup payer)", func(t *testing.T) { + chain := flow.Mainnet.Chain() + env := &fvmmock.Environment{} + env.On("Chain").Return(chain) + env.On("LimitAccountStorage").Return(true) + env.On("StartChildSpan", mock.Anything).Return( + tracing.NewMockTracerSpan()) + env.On("GetStorageUsed", mock.Anything).Return(uint64(100), nil) + env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything).Return( + cadence.NewArray([]cadence.Value{ + bytesToUFix64(100), + }), + nil, + ) + + d := &fvm.TransactionStorageLimiter{} + err := d.CheckStorageLimits(env, txnState, owner, 0) require.NoError(t, err, "Transaction with equal capacity than storage used should work") }) t.Run("capacity < storage -> Not OK", func(t *testing.T) { @@ -70,7 +105,57 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, []flow.Address{owner}, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, txnState, flow.EmptyAddress, 0) + require.Error(t, err, "Transaction with lower capacity than storage used should fail") + }) + t.Run("capacity > storage -> OK (payer not updated)", func(t *testing.T) { + chain := flow.Mainnet.Chain() + env := &fvmmock.Environment{} + env.On("Chain").Return(chain) + env.On("LimitAccountStorage").Return(true) + env.On("StartChildSpan", mock.Anything).Return( + tracing.NewMockTracerSpan()) + env.On("GetStorageUsed", mock.Anything).Return(uint64(99), nil) + env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything).Return( + cadence.NewArray([]cadence.Value{ + bytesToUFix64(100), + }), + nil, + ) + + txnState := state.NewTransactionState( + utils.NewSimpleView(), + state.DefaultParameters()) + // sanity check + require.Empty(t, txnState.UpdatedRegisterIDs()) + + d := &fvm.TransactionStorageLimiter{} + err := d.CheckStorageLimits(env, txnState, owner, 1) + require.NoError(t, err, "Transaction with higher capacity than storage used should work") + }) + t.Run("capacity < storage -> Not OK (payer not updated)", func(t *testing.T) { + chain := flow.Mainnet.Chain() + env := &fvmmock.Environment{} + env.On("Chain").Return(chain) + env.On("LimitAccountStorage").Return(true) + env.On("StartChildSpan", mock.Anything).Return( + tracing.NewMockTracerSpan()) + env.On("GetStorageUsed", mock.Anything).Return(uint64(101), nil) + env.On("AccountsStorageCapacity", mock.Anything, mock.Anything, mock.Anything).Return( + cadence.NewArray([]cadence.Value{ + bytesToUFix64(100), + }), + nil, + ) + + txnState := state.NewTransactionState( + utils.NewSimpleView(), + state.DefaultParameters()) + // sanity check + require.Empty(t, txnState.UpdatedRegisterIDs()) + + d := &fvm.TransactionStorageLimiter{} + err := d.CheckStorageLimits(env, txnState, owner, 1000) require.Error(t, err, "Transaction with lower capacity than storage used should fail") }) t.Run("if ctx LimitAccountStorage false-> OK", func(t *testing.T) { @@ -90,7 +175,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, []flow.Address{owner}, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, txnState, flow.EmptyAddress, 0) require.NoError(t, err, "Transaction with higher capacity than storage used should work") }) t.Run("non existing accounts or any other errors on fetching storage used -> Not OK", func(t *testing.T) { @@ -109,7 +194,7 @@ func TestTransactionStorageLimiter(t *testing.T) { ) d := &fvm.TransactionStorageLimiter{} - err := d.CheckStorageLimits(env, []flow.Address{owner}, flow.EmptyAddress, 0) + err := d.CheckStorageLimits(env, txnState, flow.EmptyAddress, 0) require.Error(t, err, "check storage used on non existing account (not general registers) should fail") }) } diff --git a/fvm/utils/cadenceValues.go b/fvm/utils/cadenceValues.go index 8e5abb1ba0c..41f7428d198 100644 --- a/fvm/utils/cadenceValues.go +++ b/fvm/utils/cadenceValues.go @@ -23,7 +23,10 @@ func AddressSliceToCadenceValue(addresses []common.Address) cadence.Value { return cadence.NewArray(adds) } -func CadenceValueToAddressSlice(value cadence.Value) (addresses []common.Address, ok bool) { +func CadenceValueToAddressSlice(value cadence.Value) ( + addresses []flow.Address, + ok bool, +) { // cast to array v, ok := value.(cadence.Array) @@ -37,7 +40,7 @@ func CadenceValueToAddressSlice(value cadence.Value) (addresses []common.Address if !ok { return nil, false } - addresses = append(addresses, common.Address(a)) + addresses = append(addresses, flow.ConvertAddress(a)) } return addresses, true } diff --git a/fvm/utils/view.go b/fvm/utils/view.go index 964ff9f9d70..a1e14db7cc4 100644 --- a/fvm/utils/view.go +++ b/fvm/utils/view.go @@ -41,7 +41,7 @@ func (v *SimpleView) MergeView(o state.View) error { } for key, value := range other.Ledger.Registers { - err := v.Ledger.Set(key.Owner, key.Key, value) + err := v.Ledger.Set(key, value) if err != nil { return fmt.Errorf("can not merge: %w", err) } @@ -57,12 +57,12 @@ func (v *SimpleView) DropDelta() { v.Ledger.Registers = make(map[flow.RegisterID]flow.RegisterValue) } -func (v *SimpleView) Set(owner, key string, value flow.RegisterValue) error { - return v.Ledger.Set(owner, key, value) +func (v *SimpleView) Set(id flow.RegisterID, value flow.RegisterValue) error { + return v.Ledger.Set(id, value) } -func (v *SimpleView) Get(owner, key string) (flow.RegisterValue, error) { - value, err := v.Ledger.Get(owner, key) +func (v *SimpleView) Get(id flow.RegisterID) (flow.RegisterValue, error) { + value, err := v.Ledger.Get(id) if err != nil { return nil, err } @@ -71,14 +71,14 @@ func (v *SimpleView) Get(owner, key string) (flow.RegisterValue, error) { } if v.Parent != nil { - return v.Parent.Get(owner, key) + return v.Parent.Get(id) } return nil, nil } -// returns all the registers that has been touched -func (v *SimpleView) AllRegisters() []flow.RegisterID { +// returns all the register ids that has been touched +func (v *SimpleView) AllRegisterIDs() []flow.RegisterID { res := make([]flow.RegisterID, 0, len(v.Ledger.RegisterTouches)) for k := range v.Ledger.RegisterTouches { res = append(res, k) @@ -86,6 +86,15 @@ func (v *SimpleView) AllRegisters() []flow.RegisterID { return res } +// returns all the register ids that has been updated +func (v *SimpleView) UpdatedRegisterIDs() []flow.RegisterID { + res := make([]flow.RegisterID, 0, len(v.Ledger.RegisterUpdated)) + for k := range v.Ledger.RegisterUpdated { + res = append(res, k) + } + return res +} + func (v *SimpleView) UpdatedRegisters() flow.RegisterEntries { entries := make(flow.RegisterEntries, 0, len(v.Ledger.RegisterUpdated)) for key := range v.Ledger.RegisterUpdated { @@ -99,14 +108,6 @@ func (v *SimpleView) UpdatedRegisters() flow.RegisterEntries { return entries } -func (v *SimpleView) Touch(owner, key string) error { - return v.Ledger.Touch(owner, key) -} - -func (v *SimpleView) Delete(owner, key string) error { - return v.Ledger.Delete(owner, key) -} - func (v *SimpleView) Payloads() []ledger.Payload { return v.Ledger.Payloads() } @@ -141,10 +142,9 @@ func NewMapLedgerFromPayloads(payloads []ledger.Payload) *MapLedger { panic(err) } - id := flow.RegisterID{ - Owner: string(key.KeyParts[0].Value), - Key: string(key.KeyParts[1].Value), - } + id := flow.NewRegisterID( + string(key.KeyParts[0].Value), + string(key.KeyParts[1].Value)) ledger.Registers[id] = entry.Value() } @@ -152,40 +152,22 @@ func NewMapLedgerFromPayloads(payloads []ledger.Payload) *MapLedger { return ledger } -func (m *MapLedger) Set(owner, key string, value flow.RegisterValue) error { - m.Lock() - defer m.Unlock() - - k := flow.RegisterID{Owner: owner, Key: key} - m.RegisterTouches[k] = struct{}{} - m.RegisterUpdated[k] = struct{}{} - m.Registers[k] = value - return nil -} - -func (m *MapLedger) Get(owner, key string) (flow.RegisterValue, error) { +func (m *MapLedger) Set(id flow.RegisterID, value flow.RegisterValue) error { m.Lock() defer m.Unlock() - k := flow.RegisterID{Owner: owner, Key: key} - m.RegisterTouches[k] = struct{}{} - return m.Registers[k], nil -} - -func (m *MapLedger) Touch(owner, key string) error { - m.Lock() - defer m.Unlock() - - m.RegisterTouches[flow.RegisterID{Owner: owner, Key: key}] = struct{}{} + m.RegisterTouches[id] = struct{}{} + m.RegisterUpdated[id] = struct{}{} + m.Registers[id] = value return nil } -func (m *MapLedger) Delete(owner, key string) error { +func (m *MapLedger) Get(id flow.RegisterID) (flow.RegisterValue, error) { m.Lock() defer m.Unlock() - delete(m.RegisterTouches, flow.RegisterID{Owner: owner, Key: key}) - return nil + m.RegisterTouches[id] = struct{}{} + return m.Registers[id], nil } func registerIdToLedgerKey(id flow.RegisterID) ledger.Key { diff --git a/go.mod b/go.mod index 151f73583e3..506ab806893 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,14 @@ module github.com/onflow/flow-go go 1.19 require ( - cloud.google.com/go/compute v1.6.1 + cloud.google.com/go/compute/metadata v0.2.1 cloud.google.com/go/profiler v0.3.0 - cloud.google.com/go/storage v1.22.1 + cloud.google.com/go/storage v1.27.0 github.com/antihax/optional v1.0.0 github.com/aws/aws-sdk-go-v2/config v1.8.0 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.15.0 - github.com/btcsuite/btcd v0.22.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 - github.com/dapperlabs/testingdock v0.4.4 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/ef-ds/deque v1.0.4 @@ -22,8 +20,8 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 - github.com/google/go-cmp v0.5.8 - github.com/google/pprof v0.0.0-20220818150347-1763105d910c + github.com/google/go-cmp v0.5.9 + github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 @@ -32,51 +30,50 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru v0.5.4 - github.com/improbable-eng/grpc-web v0.12.0 - github.com/ipfs/go-bitswap v0.9.0 // indirect + github.com/improbable-eng/grpc-web v0.15.0 github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-blockservice v0.4.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.2.0 - github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-provider v0.7.0 github.com/ipfs/go-ipld-format v0.3.0 github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/libp2p/go-addr-util v0.1.0 - github.com/libp2p/go-libp2p v0.23.3 - github.com/libp2p/go-libp2p-kad-dht v0.18.0 - github.com/libp2p/go-libp2p-kbucket v0.4.7 + github.com/libp2p/go-libp2p v0.24.2 + github.com/libp2p/go-libp2p-kad-dht v0.19.0 + github.com/libp2p/go-libp2p-kbucket v0.5.0 github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e - github.com/libp2p/go-libp2p-tls v0.4.1 github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c - github.com/multiformats/go-base32 v0.1.0 // indirect - github.com/multiformats/go-multiaddr v0.7.0 + github.com/montanaflynn/stats v0.6.6 + github.com/multiformats/go-multiaddr v0.8.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.1 github.com/onflow/atree v0.4.0 - github.com/onflow/cadence v0.31.2 - github.com/onflow/flow v0.3.2 - github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 - github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 - github.com/onflow/flow-go-sdk v0.31.2 + github.com/onflow/cadence v0.31.3 + github.com/onflow/flow v0.3.4 + github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 + github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 + github.com/onflow/flow-go-sdk v0.31.3 github.com/onflow/flow-go/crypto v0.24.4 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 + github.com/pkg/profile v1.7.0 github.com/prometheus/client_golang v1.14.0 github.com/rs/cors v1.8.0 - github.com/rs/zerolog v1.28.0 + github.com/rs/zerolog v1.29.0 github.com/schollz/progressbar/v3 v3.8.3 github.com/sethvargo/go-retry v0.2.3 github.com/shirou/gopsutil/v3 v3.22.2 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.11 go.opentelemetry.io/otel v1.8.0 @@ -84,35 +81,29 @@ require ( go.opentelemetry.io/otel/sdk v1.8.0 go.opentelemetry.io/otel/trace v1.8.0 go.uber.org/atomic v1.10.0 - go.uber.org/multierr v1.8.0 - golang.org/x/crypto v0.1.0 - golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.2.0 - golang.org/x/text v0.4.0 + go.uber.org/multierr v1.9.0 + golang.org/x/crypto v0.4.0 + golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 + golang.org/x/sync v0.1.0 + golang.org/x/sys v0.3.0 + golang.org/x/text v0.5.0 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - golang.org/x/tools v0.2.0 - google.golang.org/api v0.81.0 - google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd - google.golang.org/grpc v1.46.2 + golang.org/x/tools v0.4.0 + google.golang.org/api v0.102.0 + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 + google.golang.org/grpc v1.52.3 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 google.golang.org/protobuf v1.28.1 gotest.tools v2.2.0+incompatible pgregory.net/rapid v0.4.7 ) -require ( - github.com/montanaflynn/stats v0.6.6 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 - github.com/pkg/profile v1.7.0 -) +require github.com/slok/go-http-metrics v0.10.0 require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/iam v0.3.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect - github.com/Microsoft/go-winio v0.4.16 // indirect - github.com/Microsoft/hcsshim v0.8.7 // indirect + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect github.com/aws/aws-sdk-go-v2 v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect @@ -126,26 +117,19 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.0 // indirect + github.com/btcsuite/btcd v0.22.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect - github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 // indirect - github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00 // indirect - github.com/coreos/go-systemd/v22 v22.4.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect - github.com/docker/cli v0.0.0-20191105005515-99c5edceb48d // indirect - github.com/docker/distribution v2.6.0-rc.1.0.20171207180435-f4118485915a+incompatible // indirect - github.com/docker/docker v1.4.2-0.20190513124817-8c8457b0f2f8 // indirect - github.com/docker/docker-credential-helpers v0.6.3 // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect @@ -169,17 +153,19 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/googleapis/gax-go/v2 v2.4.0 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huin/goupnp v1.0.3 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-bitswap v0.9.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-fetcher v1.5.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.2 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect @@ -193,8 +179,8 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect - github.com/klauspost/compress v1.15.10 // indirect - github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/klauspost/compress v1.15.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.2 // indirect github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -204,16 +190,16 @@ require ( github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect - github.com/libp2p/go-netroute v0.2.0 // indirect + github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/libp2p/go-reuseport v0.2.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/lucas-clemente/quic-go v0.29.1 // indirect + github.com/lucas-clemente/quic-go v0.31.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect + github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect @@ -226,21 +212,17 @@ require ( github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base36 v0.1.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.6.0 // indirect + github.com/multiformats/go-multicodec v0.7.0 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect - github.com/nxadm/tail v1.4.8 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.5.0 // indirect github.com/onflow/sdks v0.4.4 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/opencontainers/runc v0.1.1 // indirect + github.com/onsi/ginkgo/v2 v2.6.1 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -249,18 +231,17 @@ require ( github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a // indirect - github.com/sirupsen/logrus v1.8.1 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect @@ -272,23 +253,23 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect go.opentelemetry.io/proto/otlp v0.18.0 // indirect - go.uber.org/zap v1.23.0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect + go.uber.org/dig v1.15.0 // indirect + go.uber.org/fx v1.18.2 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/oauth2 v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.8.2 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.66.6 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect + nhooyr.io/websocket v1.8.6 // indirect ) - -replace mellium.im/sasl => github.com/mellium/sasl v0.2.1 diff --git a/go.sum b/go.sum index 93eadab4bf2..0539b038eb3 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,3 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -31,8 +30,9 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -43,12 +43,17 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/profiler v0.3.0/go.mod h1:9wYk9eY4iZHsev8TQb61kh3wiOiSyz/xOYixWPzweCU= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -61,8 +66,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -74,8 +80,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= @@ -91,11 +95,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -165,7 +164,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.3.0 h1:h7mv5q31cthBTd7V4kLAZaIThj1e8vPGcSqpPue9KVI= github.com/bits-and-blooms/bitset v1.3.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -194,6 +192,7 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -202,8 +201,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -223,21 +222,9 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 h1:kIFnQBO7rQ0XkMe6xEwbybYHBEaWmh/f++laI6Emt7M= -github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00 h1:lsjC5ENBl+Zgf38+B0ymougXFp0BaubeIVETltYZTQw= -github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -248,8 +235,8 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -260,8 +247,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= -github.com/dapperlabs/testingdock v0.4.4 h1:nDpnEjhs2gNv7rcb70PTfHlL3yr4eQycqp0+oFuhyNg= -github.com/dapperlabs/testingdock v0.4.4/go.mod h1:HeTbuHG1J4yt4n7NlZSyuk5c5fmyz6hECbyV+36Ku7Q= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -292,19 +277,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/cli v0.0.0-20191105005515-99c5edceb48d h1:SknEFm9d070Wn2GeX8dyl7bMrX07cp3UMXuZ2Ct02Kw= -github.com/docker/cli v0.0.0-20191105005515-99c5edceb48d/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v2.6.0-rc.1.0.20171207180435-f4118485915a+incompatible h1:2YJcZ66ScSWjLY7lifaPjEav51u0EThWBHpfveH6p0g= -github.com/docker/distribution v2.6.0-rc.1.0.20171207180435-f4118485915a+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190513124817-8c8457b0f2f8 h1:vyqIlE9fpJ+cdE95qkW9ihHas6QT87AFLE72W5bGUEY= -github.com/docker/docker v1.4.2-0.20190513124817-8c8457b0f2f8/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -368,8 +341,11 @@ github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -381,8 +357,6 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -398,8 +372,15 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -407,7 +388,12 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -483,8 +469,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -514,8 +501,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20220412212628-83db2b799d1f/go.mod h1:Pt31oes+eGImORns3McJn8zHefuQl2rG8l6xQjGYB4U= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c h1:jk3Bnv5QzOSJil2OC6EQBoqy+lj/Kw56nrkEzvrQm2w= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -523,6 +510,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -531,9 +520,9 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= +github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -554,6 +543,7 @@ github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 h1:uxUHSMwWDJ/9jVPHNumRC8WZOi3hrBL22ObVOoLg4ww= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2/go.mod h1:BL7w7qd2l/j9jgY6WMhYutfOFQc0I8RTVwtjpnAMoTM= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea h1:1Tk1IbruXbunEnaIZEFb+Hpv9BIZti3OxKwKn5wWyKk= @@ -570,14 +560,12 @@ github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfm github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= @@ -609,10 +597,11 @@ github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/improbable-eng/grpc-web v0.12.0 h1:GlCS+lMZzIkfouf7CNqY+qqpowdKuJLSLLcKVfM1oLc= -github.com/improbable-eng/grpc-web v0.12.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= +github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= @@ -758,9 +747,9 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -779,15 +768,17 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= -github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= -github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= +github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -809,8 +800,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/lib/pq v0.0.0-20170810061220-e42267488fe3 h1:2Fs7SMFLrtkGta5HodD3MRV3nIzv+6I90eSfqwPklbo= -github.com/lib/pq v0.0.0-20170810061220-e42267488fe3/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= @@ -837,9 +828,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.23.3 h1:/n3i0VtJF0iZ9YMUxl/teOY3h+M8NfgaCjOSYr9D+uI= -github.com/libp2p/go-libp2p v0.23.3/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= +github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= +github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -873,7 +863,6 @@ github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqe github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= @@ -886,10 +875,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.18.0 h1:akqO3gPMwixR7qFSFq70ezRun97g5hrA/lBW9jrjUYM= -github.com/libp2p/go-libp2p-kad-dht v0.18.0/go.mod h1:Gb92MYIPm3K2pJLGn8wl0m8wiKDvHrYpg+rOd0GzzPA= -github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= -github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= +github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -913,7 +902,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.8.0 h1:bzTG693TA1Ju/zKmUCQzDLSqiJnyRFVwPpuloZ/OZtI= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e h1:phmi6mEoO5y2AQP68+4vZhNpHtZ4dum2ieFtWdmjXak= github.com/libp2p/go-libp2p-pubsub v0.8.2-0.20221201175637-3d2eab35722e/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= @@ -942,8 +930,6 @@ github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehts github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-tls v0.4.1 h1:1ByJUbyoMXvYXDoW6lLsMxqMViQNXmt+CfQqlnCpY+M= -github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= @@ -982,8 +968,8 @@ github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdm github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= -github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= -github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -1030,8 +1016,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.29.1 h1:Z+WMJ++qMLhvpFkRZA+jl3BTxUjm415YBmWanXB8zP0= -github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= +github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= +github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -1043,17 +1029,17 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= +github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= +github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.1.1 h1:TnyKp3pEXcDooTaNn4s9dYpMJ7kMnTp7k5h+SgYP/mc= +github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -1069,6 +1055,7 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= @@ -1119,14 +1106,13 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ= github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -1137,8 +1123,9 @@ github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -1150,8 +1137,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= -github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1173,8 +1160,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= -github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= +github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1196,11 +1183,13 @@ github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcR github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -1213,8 +1202,6 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -1223,22 +1210,22 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onflow/atree v0.4.0 h1:+TbNisavAkukAKhgQ4plWnvR9o5+SkwPIsi3jaeAqKs= github.com/onflow/atree v0.4.0/go.mod h1:7Qe1xaW0YewvouLXrugzMFUYXNoRQ8MT/UsVAWx1Ndo= -github.com/onflow/cadence v0.31.2 h1:13OTvfyACXyR3pv0Lkp2IAjXWMIXMee7Bn2yFz0onrM= -github.com/onflow/cadence v0.31.2/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= -github.com/onflow/flow v0.3.2 h1:z3IuKOjM9Tvf0pXfloTbrLxM5nTunI47cklsDd+wxBE= -github.com/onflow/flow v0.3.2/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 h1:fH5e7L9xFXNOd3WLvMaPNkP1m7BngRTDP751zMNndws= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:9nrgjIF/noY2jJ7LP8bKLHTpcdHOa2yO0ryCKTQpxvs= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 h1:vgNS6I+MM/74pciIoKb7ZBs8XGF1ONsSdkAec36B9iU= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:WMmeggH/H9Xb/SsT+4QFMtGYf+p1S2LXzbZAIaQQWAk= +github.com/onflow/cadence v0.31.3 h1:lTHTBnoFi/ahGk1cQ9JAC+tfYir4kjRVIqtTVCjbe6E= +github.com/onflow/cadence v0.31.3/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= +github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= +github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-go-sdk v0.31.2 h1:DuTzsNjheaL5r9IQWtGC08Kq5JMZ4Uf2OA/Pst6bsSg= -github.com/onflow/flow-go-sdk v0.31.2/go.mod h1:l9Fgfg+aBx4ILvEE1Wf6g5G9Lg1A+hzY6oh4bQdkcFs= +github.com/onflow/flow-go-sdk v0.31.3 h1:CytMRiTayXRlkRNXQ9Cw6ZcKoOIK6+QtXk4iUb8f0zQ= +github.com/onflow/flow-go-sdk v0.31.3/go.mod h1:cqj2QShwC4DqxWzrg0+U7KxE2k7OJDGBxh8XZrJ4v5E= github.com/onflow/flow-go/crypto v0.24.4 h1:SwEtoVS2TidCIHYCZMgQ7U2YsqhI9upnw94fhdHTubM= github.com/onflow/flow-go/crypto v0.24.4/go.mod h1:dkVL98P6GHR48iD9zCB6XlnkJX8IQd00FKgt1reV90w= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310 h1:fOxDpxEi4XbOql9eMc2wBFil21TOdbnjuDjePgnI3Uc= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 h1:haWv3D5loiH+zcOoWEvDXtWQvXt5U8PLliQjwhv9sfw= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.4.4 h1:aJPGJJLAN+mlBWAQxsyuJXeRRMFeLwU6Mp4e/YL6bdU= @@ -1246,33 +1233,21 @@ github.com/onflow/sdks v0.4.4/go.mod h1:F0dj0EyHC55kknLkeD10js4mo14yTdMotnWMslPi github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1302,7 +1277,6 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1322,12 +1296,9 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1343,27 +1314,23 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= @@ -1381,14 +1348,15 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= -github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1425,14 +1393,13 @@ github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go. github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= +github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -1455,15 +1422,13 @@ github.com/spf13/afero v1.9.0/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcD github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -1481,8 +1446,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1491,13 +1457,13 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= github.com/supranational/blst v0.3.4 h1:iZE9lBMoywK2uy2U/5hDOvobQk9FnOQ2wNlu9GmRCoA= github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -1513,10 +1479,11 @@ github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d h1:5JInRQbk5UBX github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d/go.mod h1:Nlx5Y115XQvNcIdIy7dZXaNSUpzwBSge4/Ivk93/Yog= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1547,9 +1514,6 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+m github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1578,8 +1542,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= @@ -1602,6 +1567,10 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= +go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -1610,8 +1579,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1619,12 +1588,11 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1659,8 +1627,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1674,8 +1642,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1703,8 +1672,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1741,6 +1710,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1760,16 +1730,15 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1790,8 +1759,9 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1804,9 +1774,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1829,14 +1798,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1847,7 +1814,6 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1861,6 +1827,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1876,7 +1843,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1893,12 +1859,12 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1910,24 +1876,22 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1937,8 +1901,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2004,7 +1968,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2015,16 +1978,15 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= @@ -2072,8 +2034,8 @@ google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.81.0 h1:o8WF5AvfidafWbFjsRyupxyEQJNUWxLZJCK5NXrxZZ8= -google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2110,6 +2072,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -2125,6 +2088,7 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2166,8 +2130,8 @@ google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2189,6 +2153,7 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= @@ -2206,8 +2171,8 @@ google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2226,7 +2191,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2237,7 +2201,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -2248,7 +2211,6 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -2277,10 +2239,11 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= +nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/insecure/corrupt_conduit_factory.go b/insecure/corrupt_conduit_factory.go index f36fc3439b1..8242cc5479b 100644 --- a/insecure/corrupt_conduit_factory.go +++ b/insecure/corrupt_conduit_factory.go @@ -13,11 +13,11 @@ type CorruptConduitFactory interface { // through the specified protocol to the target identifiers. SendOnFlowNetwork(interface{}, channels.Channel, Protocol, uint, ...flow.Identifier) error - // UnregisterChannel is called by the slave conduits of this factory to let it know that the corresponding engine of the + // UnregisterChannel is called by the secondary conduits of this factory to let it know that the corresponding engine of the // conduit is not going to use it anymore, so the channel can be closed safely. UnregisterChannel(channels.Channel) error - // RegisterEgressController sets the EgressController component of the factory. All outgoing messages of the (slave) conduits that + // RegisterEgressController sets the EgressController component of the factory. All outgoing messages of the (secondary) conduits that // this factory creates are forwarded to the EgressController instead of being dispatched on the Flow network. RegisterEgressController(EgressController) error } diff --git a/insecure/corruptlibp2p/libp2p_node_factory.go b/insecure/corruptlibp2p/libp2p_node_factory.go index 67e7a35694f..61ca08afea2 100644 --- a/insecure/corruptlibp2p/libp2p_node_factory.go +++ b/insecure/corruptlibp2p/libp2p_node_factory.go @@ -2,21 +2,22 @@ package corruptlibp2p import ( "context" + "fmt" "time" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" - "github.com/onflow/flow-go/network/p2p" - madns "github.com/multiformats/go-multiaddr-dns" "github.com/rs/zerolog" fcrypto "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/p2pbuilder" + "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" ) // NewCorruptLibP2PNodeFactory wrapper around the original DefaultLibP2PNodeFactory. Nodes returned from this factory func will be corrupted libp2p nodes. @@ -44,7 +45,7 @@ func NewCorruptLibP2PNodeFactory( panic("illegal chain id for using corrupt libp2p node") } - builder := p2pbuilder.DefaultNodeBuilder( + builder, err := p2pbuilder.DefaultNodeBuilder( log, address, flowKey, @@ -58,7 +59,12 @@ func NewCorruptLibP2PNodeFactory( peerScoringEnabled, connectionPruning, updateInterval, - p2pbuilder.DefaultResourceManagerConfig()) + p2pbuilder.DefaultResourceManagerConfig(), + ratelimit.NewUnicastRateLimiterDistributor()) + + if err != nil { + return nil, fmt.Errorf("could not create corrupt libp2p node builder: %w", err) + } if topicValidatorDisabled { builder.SetCreateNode(NewCorruptLibP2PNode) } diff --git a/insecure/corruptlibp2p/p2p_node.go b/insecure/corruptlibp2p/p2p_node.go index b2590449cd5..2f3a9e4115c 100644 --- a/insecure/corruptlibp2p/p2p_node.go +++ b/insecure/corruptlibp2p/p2p_node.go @@ -22,31 +22,18 @@ import ( // AcceptAllTopicValidator pubsub validator func that does not perform any validation, it will only attempt to decode the message and update the // rawMsg.ValidatorData needed for further processing by the middleware receive loop. Malformed messages that fail to unmarshal or decode will result // in a pubsub.ValidationReject result returned. -func AcceptAllTopicValidator(lg zerolog.Logger, c network.Codec) p2p.TopicValidatorFunc { - lg = lg.With(). - Str("component", "corrupt-libp2p-node-topic-validator"). - Logger() - - return func(ctx context.Context, from peer.ID, rawMsg *pubsub.Message) p2p.ValidationResult { +func AcceptAllTopicValidator() p2p.TopicValidatorFunc { + return func(_ context.Context, from peer.ID, rawMsg *pubsub.Message) p2p.ValidationResult { var msg message.Message err := msg.Unmarshal(rawMsg.Data) if err != nil { - lg.Err(err).Msg("could not unmarshal raw message data") - return p2p.ValidationReject - } - - decodedMsgPayload, err := c.Decode(msg.Payload) - if err != nil { - lg.Err(err).Msg("could not decode message payload") return p2p.ValidationReject } rawMsg.ValidatorData = validator.TopicValidatorData{ - Message: &msg, - DecodedMsgPayload: decodedMsgPayload, - From: from, + Message: &msg, + From: from, } - return p2p.ValidationAccept } } @@ -61,7 +48,7 @@ type CorruptP2PNode struct { // Subscribe subscribes the node to the given topic with a noop topic validator. // All errors returned from this function can be considered benign. func (n *CorruptP2PNode) Subscribe(topic channels.Topic, _ p2p.TopicValidatorFunc) (p2p.Subscription, error) { - topicValidator := AcceptAllTopicValidator(n.logger, n.codec) + topicValidator := AcceptAllTopicValidator() return n.Node.Subscribe(topic, topicValidator) } diff --git a/insecure/corruptnet/conduit_factory.go b/insecure/corruptnet/conduit_factory.go index 65939c4d572..c62ab0b2340 100644 --- a/insecure/corruptnet/conduit_factory.go +++ b/insecure/corruptnet/conduit_factory.go @@ -83,7 +83,7 @@ func (c *ConduitFactory) NewConduit(ctx context.Context, channel channels.Channe return con, nil } -// UnregisterChannel is called by the slave conduits of this factory to let it know that the corresponding engine of the +// UnregisterChannel is called by the secondary conduits of this factory to let it know that the corresponding engine of the // conduit is not going to use it anymore, so the channel can be closed safely. func (c *ConduitFactory) UnregisterChannel(channel channels.Channel) error { return c.adapter.UnRegisterChannel(channel) diff --git a/insecure/go.mod b/insecure/go.mod index a76c78aabf1..68f36aaef41 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -6,24 +6,25 @@ require ( github.com/golang/protobuf v1.5.2 github.com/hashicorp/go-multierror v1.1.1 github.com/ipfs/go-datastore v0.6.0 - github.com/libp2p/go-libp2p v0.23.3 + github.com/libp2p/go-libp2p v0.24.2 github.com/libp2p/go-libp2p-pubsub v0.8.2 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/onflow/flow-go v0.0.0-00010101000000-000000000000 + github.com/onflow/flow-go v0.29.8 github.com/onflow/flow-go/crypto v0.24.4 - github.com/rs/zerolog v1.28.0 + github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee - google.golang.org/grpc v1.50.1 + google.golang.org/grpc v1.52.3 google.golang.org/protobuf v1.28.1 ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.6.1 // indirect - cloud.google.com/go/iam v0.3.0 // indirect - cloud.google.com/go/storage v1.22.1 // indirect + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/storage v1.27.0 // indirect github.com/aws/aws-sdk-go-v2 v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect @@ -44,9 +45,9 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect - github.com/coreos/go-systemd/v22 v22.4.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect @@ -82,12 +83,12 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20220818150347-1763105d910c // indirect + github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/gax-go/v2 v2.4.0 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 // indirect @@ -98,8 +99,8 @@ require ( github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huin/goupnp v1.0.3 // indirect - github.com/improbable-eng/grpc-web v0.12.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/improbable-eng/grpc-web v0.15.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect @@ -127,8 +128,8 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect - github.com/klauspost/compress v1.15.10 // indirect - github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/klauspost/compress v1.15.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.2 // indirect github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-addr-util v0.1.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -136,23 +137,22 @@ require ( github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-core v0.20.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.18.0 // indirect - github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-tls v0.4.1 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect - github.com/libp2p/go-netroute v0.2.0 // indirect + github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/libp2p/go-reuseport v0.2.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/lucas-clemente/quic-go v0.29.1 // indirect + github.com/lucas-clemente/quic-go v0.31.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect + github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect @@ -167,25 +167,24 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect - github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multiaddr v0.7.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.6.0 // indirect + github.com/multiformats/go-multicodec v0.7.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect - github.com/nxadm/tail v1.4.8 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.4.0 // indirect - github.com/onflow/cadence v0.31.2 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 // indirect + github.com/onflow/cadence v0.31.3 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.5.0 // indirect - github.com/onflow/flow-go-sdk v0.31.2 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310 // indirect + github.com/onflow/flow-go-sdk v0.31.3 // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 // indirect github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 // indirect github.com/onflow/sdks v0.4.4 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/ginkgo/v2 v2.6.1 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -198,8 +197,8 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a // indirect @@ -207,14 +206,15 @@ require ( github.com/schollz/progressbar/v3 v3.8.3 // indirect github.com/sethvargo/go-retry v0.2.3 // indirect github.com/shirou/gopsutil/v3 v3.22.2 // indirect + github.com/slok/go-http-metrics v0.10.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.0 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.5.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.12.0 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect @@ -228,7 +228,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect @@ -237,29 +237,31 @@ require ( go.opentelemetry.io/otel/trace v1.8.0 // indirect go.opentelemetry.io/proto/otlp v0.18.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.23.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + go.uber.org/dig v1.15.0 // indirect + go.uber.org/fx v1.18.2 // indirect + go.uber.org/multierr v1.9.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.4.0 // indirect + golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/oauth2 v0.3.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/tools v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/api v0.81.0 // indirect + golang.org/x/tools v0.4.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.102.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect gopkg.in/ini.v1 v1.66.6 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect + nhooyr.io/websocket v1.8.6 // indirect ) replace github.com/onflow/flow-go => ../ diff --git a/insecure/go.sum b/insecure/go.sum index 42fc2698337..112661c7d18 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -19,35 +19,23 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -59,8 +47,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -72,7 +60,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= @@ -88,8 +75,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= @@ -185,6 +170,7 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -193,8 +179,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -210,15 +196,12 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 h1:kIFnQBO7rQ0XkMe6xEwbybYHBEaWmh/f++laI6Emt7M= -github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00 h1:lsjC5ENBl+Zgf38+B0ymougXFp0BaubeIVETltYZTQw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -229,8 +212,8 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -241,7 +224,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= -github.com/dapperlabs/testingdock v0.4.4 h1:nDpnEjhs2gNv7rcb70PTfHlL3yr4eQycqp0+oFuhyNg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -272,13 +254,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/cli v0.0.0-20191105005515-99c5edceb48d h1:SknEFm9d070Wn2GeX8dyl7bMrX07cp3UMXuZ2Ct02Kw= -github.com/docker/distribution v2.6.0-rc.1.0.20171207180435-f4118485915a+incompatible h1:2YJcZ66ScSWjLY7lifaPjEav51u0EThWBHpfveH6p0g= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190513124817-8c8457b0f2f8 h1:vyqIlE9fpJ+cdE95qkW9ihHas6QT87AFLE72W5bGUEY= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -303,10 +279,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.9.13 h1:rOPqjSngvs1VSYH2H+PMPiWt4VEulvNRbFgqiGqJM3E= github.com/ethereum/go-ethereum v1.9.13/go.mod h1:qwN9d1GLyDh0N7Ab8bMGd0H9knaji2jOBm2RrMGjXls= @@ -340,8 +314,11 @@ github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -353,8 +330,6 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -370,8 +345,15 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -379,6 +361,12 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -411,7 +399,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -431,7 +418,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -454,8 +440,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -467,7 +453,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -478,13 +463,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c h1:jk3Bnv5QzOSJil2OC6EQBoqy+lj/Kw56nrkEzvrQm2w= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -492,18 +472,14 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -523,6 +499,7 @@ github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 h1:uxUHSMwWDJ/9jVPHNumRC8WZOi3hrBL22ObVOoLg4ww= github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2/go.mod h1:BL7w7qd2l/j9jgY6WMhYutfOFQc0I8RTVwtjpnAMoTM= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea h1:1Tk1IbruXbunEnaIZEFb+Hpv9BIZti3OxKwKn5wWyKk= @@ -575,10 +552,11 @@ github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixH github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/improbable-eng/grpc-web v0.12.0 h1:GlCS+lMZzIkfouf7CNqY+qqpowdKuJLSLLcKVfM1oLc= -github.com/improbable-eng/grpc-web v0.12.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= +github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= @@ -723,9 +701,9 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -744,15 +722,17 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= -github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= -github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= +github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -774,6 +754,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0 h1:acKsntI33w2bTU7tC9a0SaPimJGfSI0bFKC18ChxeVI= @@ -800,9 +782,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.23.3 h1:/n3i0VtJF0iZ9YMUxl/teOY3h+M8NfgaCjOSYr9D+uI= -github.com/libp2p/go-libp2p v0.23.3/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= +github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= +github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -836,7 +817,6 @@ github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqe github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= @@ -849,10 +829,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.18.0 h1:akqO3gPMwixR7qFSFq70ezRun97g5hrA/lBW9jrjUYM= -github.com/libp2p/go-libp2p-kad-dht v0.18.0/go.mod h1:Gb92MYIPm3K2pJLGn8wl0m8wiKDvHrYpg+rOd0GzzPA= -github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= -github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= +github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -876,7 +856,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.8.0 h1:bzTG693TA1Ju/zKmUCQzDLSqiJnyRFVwPpuloZ/OZtI= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-pubsub v0.8.2 h1:QLGUmkgKmwEVxVDYGsqc5t9CykOMY2Y21cXQHjR462I= github.com/libp2p/go-libp2p-pubsub v0.8.2/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= @@ -905,8 +884,6 @@ github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehts github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-tls v0.4.1 h1:1ByJUbyoMXvYXDoW6lLsMxqMViQNXmt+CfQqlnCpY+M= -github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= @@ -945,8 +922,8 @@ github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdm github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= -github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= -github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -993,8 +970,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.29.1 h1:Z+WMJ++qMLhvpFkRZA+jl3BTxUjm415YBmWanXB8zP0= -github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= +github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= +github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -1006,17 +983,17 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= +github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= +github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.1.1 h1:TnyKp3pEXcDooTaNn4s9dYpMJ7kMnTp7k5h+SgYP/mc= +github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -1032,6 +1009,7 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= @@ -1082,11 +1060,11 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -1097,8 +1075,9 @@ github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -1110,8 +1089,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= -github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1133,8 +1112,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= -github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= +github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1156,11 +1135,13 @@ github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcR github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -1173,8 +1154,6 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -1183,20 +1162,20 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onflow/atree v0.4.0 h1:+TbNisavAkukAKhgQ4plWnvR9o5+SkwPIsi3jaeAqKs= github.com/onflow/atree v0.4.0/go.mod h1:7Qe1xaW0YewvouLXrugzMFUYXNoRQ8MT/UsVAWx1Ndo= -github.com/onflow/cadence v0.31.2 h1:13OTvfyACXyR3pv0Lkp2IAjXWMIXMee7Bn2yFz0onrM= -github.com/onflow/cadence v0.31.2/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 h1:fH5e7L9xFXNOd3WLvMaPNkP1m7BngRTDP751zMNndws= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:9nrgjIF/noY2jJ7LP8bKLHTpcdHOa2yO0ryCKTQpxvs= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 h1:vgNS6I+MM/74pciIoKb7ZBs8XGF1ONsSdkAec36B9iU= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:WMmeggH/H9Xb/SsT+4QFMtGYf+p1S2LXzbZAIaQQWAk= +github.com/onflow/cadence v0.31.3 h1:lTHTBnoFi/ahGk1cQ9JAC+tfYir4kjRVIqtTVCjbe6E= +github.com/onflow/cadence v0.31.3/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-go-sdk v0.31.2 h1:DuTzsNjheaL5r9IQWtGC08Kq5JMZ4Uf2OA/Pst6bsSg= -github.com/onflow/flow-go-sdk v0.31.2/go.mod h1:l9Fgfg+aBx4ILvEE1Wf6g5G9Lg1A+hzY6oh4bQdkcFs= +github.com/onflow/flow-go-sdk v0.31.3 h1:CytMRiTayXRlkRNXQ9Cw6ZcKoOIK6+QtXk4iUb8f0zQ= +github.com/onflow/flow-go-sdk v0.31.3/go.mod h1:cqj2QShwC4DqxWzrg0+U7KxE2k7OJDGBxh8XZrJ4v5E= github.com/onflow/flow-go/crypto v0.24.4 h1:SwEtoVS2TidCIHYCZMgQ7U2YsqhI9upnw94fhdHTubM= github.com/onflow/flow-go/crypto v0.24.4/go.mod h1:dkVL98P6GHR48iD9zCB6XlnkJX8IQd00FKgt1reV90w= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310 h1:fOxDpxEi4XbOql9eMc2wBFil21TOdbnjuDjePgnI3Uc= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221130185733-92eb85ead310/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 h1:haWv3D5loiH+zcOoWEvDXtWQvXt5U8PLliQjwhv9sfw= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.4.4 h1:aJPGJJLAN+mlBWAQxsyuJXeRRMFeLwU6Mp4e/YL6bdU= @@ -1207,19 +1186,16 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -1271,8 +1247,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1290,11 +1264,10 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1302,10 +1275,10 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= @@ -1323,14 +1296,15 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= -github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1372,6 +1346,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= +github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -1396,8 +1372,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1418,8 +1394,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1428,8 +1405,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= github.com/supranational/blst v0.3.4 h1:iZE9lBMoywK2uy2U/5hDOvobQk9FnOQ2wNlu9GmRCoA= @@ -1449,8 +1427,10 @@ github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d h1:5JInRQbk5UBX github.com/turbolent/prettier v0.0.0-20220320183459-661cc755135d/go.mod h1:Nlx5Y115XQvNcIdIy7dZXaNSUpzwBSge4/Ivk93/Yog= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1512,8 +1492,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= @@ -1536,6 +1517,10 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= +go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -1544,8 +1529,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1553,8 +1538,8 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1592,8 +1577,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1607,8 +1592,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1624,7 +1610,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1636,8 +1621,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1674,6 +1659,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1692,17 +1678,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1714,17 +1693,9 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1737,9 +1708,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1791,6 +1761,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1806,57 +1777,38 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1865,9 +1817,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1933,27 +1884,21 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= @@ -1982,26 +1927,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.81.0 h1:o8WF5AvfidafWbFjsRyupxyEQJNUWxLZJCK5NXrxZZ8= -google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2038,6 +1965,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -2053,49 +1981,11 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= -google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2117,26 +2007,16 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2152,7 +2032,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -2175,7 +2054,6 @@ gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -2207,6 +2085,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= +nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/insecure/mock/conduit_controller.go b/insecure/mock/conduit_controller.go new file mode 100644 index 00000000000..3db99ab9c3a --- /dev/null +++ b/insecure/mock/conduit_controller.go @@ -0,0 +1,67 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mockinsecure + +import ( + flow "github.com/onflow/flow-go/model/flow" + channels "github.com/onflow/flow-go/network/channels" + + insecure "github.com/onflow/flow-go/insecure" + + mock "github.com/stretchr/testify/mock" +) + +// ConduitController is an autogenerated mock type for the ConduitController type +type ConduitController struct { + mock.Mock +} + +// EngineClosingChannel provides a mock function with given fields: _a0 +func (_m *ConduitController) EngineClosingChannel(_a0 channels.Channel) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(channels.Channel) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// HandleIncomingEvent provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *ConduitController) HandleIncomingEvent(_a0 interface{}, _a1 channels.Channel, _a2 insecure.Protocol, _a3 uint32, _a4 ...flow.Identifier) error { + _va := make([]interface{}, len(_a4)) + for _i := range _a4 { + _va[_i] = _a4[_i] + } + var _ca []interface{} + _ca = append(_ca, _a0, _a1, _a2, _a3) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 error + if rf, ok := ret.Get(0).(func(interface{}, channels.Channel, insecure.Protocol, uint32, ...flow.Identifier) error); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4...) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewConduitController interface { + mock.TestingT + Cleanup(func()) +} + +// NewConduitController creates a new instance of ConduitController. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConduitController(t mockConstructorTestingTNewConduitController) *ConduitController { + mock := &ConduitController{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/integration/Makefile b/integration/Makefile index 2e0ffa8e8b1..15cc6fcb557 100644 --- a/integration/Makefile +++ b/integration/Makefile @@ -38,7 +38,8 @@ consensus-tests: .PHONY: epochs-tests epochs-tests: - go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic ./tests/epochs/... + # Use a higher timeout of 20m for the suite of tests which span full epochs + go test $(if $(VERBOSE),-v,) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(NUM_RUNS),-count $(NUM_RUNS),) -tags relic -timeout 30m ./tests/epochs/... .PHONY: ghost-tests ghost-tests: diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index cff126eb4c6..f3166e8a57d 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -37,7 +37,7 @@ func createNodes( conIdentities flow.IdentityList, currentEpochSetup flow.EpochSetup, nextEpochSetup flow.EpochSetup, - firstBlockID flow.Identifier) ([]*node, flow.IdentityList) { + firstBlock *flow.Header) ([]*node, flow.IdentityList) { // We need to initialise the nodes with a list of identities that contain // all roles, otherwise there would be an error initialising the first epoch @@ -53,7 +53,7 @@ func createNodes( whiteboard, currentEpochSetup, nextEpochSetup, - firstBlockID)) + firstBlock)) } return nodes, conIdentities @@ -70,7 +70,7 @@ func createNode( whiteboard *whiteboard, currentSetup flow.EpochSetup, nextSetup flow.EpochSetup, - firstBlock flow.Identifier) *node { + firstBlock *flow.Header) *node { core := testutil.GenericNodeFromParticipants(t, hub, id, ids, chainID) core.Log = zerolog.New(os.Stdout).Level(zerolog.WarnLevel) @@ -108,7 +108,7 @@ func createNode( snapshot.On("Phase").Return(flow.EpochPhaseStaking, nil) snapshot.On("Head").Return(firstBlock, nil) state := new(protocolmock.MutableState) - state.On("AtBlockID", firstBlock).Return(snapshot) + state.On("AtBlockID", firstBlock.ID()).Return(snapshot) state.On("Final").Return(snapshot) core.State = state @@ -242,7 +242,7 @@ func TestWithWhiteboard(t *testing.T) { conIdentities, currentEpochSetup, nextEpochSetup, - firstBlock.ID()) + firstBlock) for _, n := range nodes { n.Ready() diff --git a/integration/go.mod b/integration/go.mod index 415a40ecd68..213679f06c0 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -3,50 +3,51 @@ module github.com/onflow/flow-go/integration go 1.19 require ( - cloud.google.com/go/bigquery v1.40.0 + cloud.google.com/go/bigquery v1.43.0 github.com/VividCortex/ewma v1.2.0 github.com/dapperlabs/testingdock v0.4.4 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/docker/docker v1.4.2-0.20190513124817-8c8457b0f2f8 github.com/docker/go-connections v0.4.0 - github.com/go-git/go-git/v5 v5.4.2 + github.com/go-git/go-git/v5 v5.5.2 github.com/go-yaml/yaml v2.1.0+incompatible github.com/ipfs/go-blockservice v0.4.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.2.0 - github.com/onflow/cadence v0.31.2 - github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 - github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 - github.com/onflow/flow-emulator v0.42.1-0.20230110232744-399decb21158 - github.com/onflow/flow-ft/lib/go/templates v0.2.0 - github.com/onflow/flow-go v0.28.1-0.20230110053724-eb549a9d362f - github.com/onflow/flow-go-sdk v0.31.2 + github.com/onflow/cadence v0.31.3 + github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 + github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 + github.com/onflow/flow-emulator v0.43.1-0.20230202181019-910459a16e2e + github.com/onflow/flow-go v0.29.9 + github.com/onflow/flow-go-sdk v0.31.3 github.com/onflow/flow-go/crypto v0.24.4 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 github.com/plus3it/gorecurcopy v0.0.1 github.com/prometheus/client_golang v1.14.0 - github.com/rs/zerolog v1.28.0 - github.com/stretchr/testify v1.8.0 + github.com/rs/zerolog v1.29.0 + github.com/stretchr/testify v1.8.1 go.einride.tech/pid v0.1.0 go.uber.org/atomic v1.10.0 - golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - google.golang.org/grpc v1.50.1 + golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 + golang.org/x/sync v0.1.0 + google.golang.org/grpc v1.52.3 google.golang.org/protobuf v1.28.1 ) require ( - cloud.google.com/go v0.102.1 // indirect - cloud.google.com/go/compute v1.7.0 // indirect - cloud.google.com/go/iam v0.3.0 // indirect - cloud.google.com/go/storage v1.23.0 // indirect + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/storage v1.27.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect - github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.8.7 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect + github.com/OneOfOne/xxhash v1.2.5 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/aws/aws-sdk-go-v2 v1.9.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.8.0 // indirect @@ -68,11 +69,12 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 // indirect github.com/containerd/fifo v0.0.0-20191213151349-ff969a566b00 // indirect - github.com/coreos/go-systemd/v22 v22.4.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect @@ -87,7 +89,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/ef-ds/deque v1.0.4 // indirect github.com/elastic/gosigar v0.14.2 // indirect - github.com/emirpasic/gods v1.12.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/ethereum/go-ethereum v1.10.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -98,7 +100,7 @@ require ( github.com/gammazero/workerpool v1.1.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-billy/v5 v5.4.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -114,13 +116,12 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20220818150347-1763105d910c // indirect + github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect - github.com/googleapis/gax-go/v2 v2.5.1 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea // indirect @@ -131,8 +132,8 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huin/goupnp v1.0.3 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect @@ -157,35 +158,34 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/go-bindata v3.23.0+incompatible // indirect - github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect - github.com/klauspost/compress v1.15.10 // indirect - github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.15.13 // indirect + github.com/klauspost/cpuid/v2 v2.2.2 // indirect github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-addr-util v0.1.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect - github.com/libp2p/go-libp2p v0.23.3 // indirect + github.com/libp2p/go-libp2p v0.24.2 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-core v0.20.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.18.0 // indirect - github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-tls v0.4.1 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect - github.com/libp2p/go-netroute v0.2.0 // indirect + github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/libp2p/go-reuseport v0.2.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/lucas-clemente/quic-go v0.29.1 // indirect + github.com/lucas-clemente/quic-go v0.31.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/m4ksio/wal v1.0.1-0.20221209164835-154a17396e4c // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect + github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect @@ -197,26 +197,24 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect - github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multiaddr v0.7.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.6.0 // indirect + github.com/multiformats/go-multicodec v0.7.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect - github.com/nxadm/tail v1.4.8 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.4.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.5.0 // indirect github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 // indirect github.com/onflow/sdks v0.4.4 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/ginkgo/v2 v2.6.1 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect @@ -226,13 +224,14 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pjbgf/sha1cd v0.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/psiemens/sconfig v0.1.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a // indirect @@ -241,15 +240,17 @@ require ( github.com/sethvargo/go-retry v0.2.3 // indirect github.com/shirou/gopsutil/v3 v3.22.2 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/skeema/knownhosts v1.1.0 // indirect + github.com/slok/go-http-metrics v0.10.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.0 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.5.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.12.0 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect @@ -261,11 +262,11 @@ require ( github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/xanzy/ssh-agent v0.3.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.2-0.20221208234712-b44d9133e4ee // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect @@ -273,33 +274,31 @@ require ( go.opentelemetry.io/otel/sdk v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect go.opentelemetry.io/proto/otlp v0.18.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.23.0 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/term v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + go.uber.org/dig v1.15.0 // indirect + go.uber.org/fx v1.18.2 // indirect + go.uber.org/multierr v1.9.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.4.0 // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/oauth2 v0.3.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/tools v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/api v0.94.0 // indirect + golang.org/x/tools v0.4.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.102.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220902135211-223410557253 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect gopkg.in/ini.v1 v1.66.6 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) -// temp fix for MacOS build. See comment https://github.com/ory/dockertest/issues/208#issuecomment-686820414 -//replace golang.org/x/sys => golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6 - replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure diff --git a/integration/go.sum b/integration/go.sum index f8e36044510..1f14a0cb615 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -22,43 +22,28 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.40.0 h1:ZmiuWZWQEZ8WphuA1J6SGV9t+XQEdwWa4+9joutHo6U= -cloud.google.com/go/bigquery v1.40.0/go.mod h1:V9NIK7zJWZzxBMSeZJoNJWqinqlL4g0eV8Y9UtDuHOI= +cloud.google.com/go/bigquery v1.43.0 h1:u0fvz5ysJBe1jwUPI4LuPwAX+o+6fCUwf3ECeg6eDUQ= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/datacatalog v1.3.0 h1:3llKXv7cC1acsWjvWmG0NQQkYVSVgunMSfVk7h6zz8Q= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/datacatalog v1.8.0 h1:6kZ4RIOW/uT7QWC5SfPfq/G8sYzr/v+UOmOAxy4Z1TE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -70,9 +55,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0 h1:wWRIaDURQA8xxHguFCshYepGlrWIrbBnAmc7wfg07qY= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -103,21 +87,19 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= -github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= +github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -134,8 +116,9 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -217,8 +200,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/c-bata/go-prompt v0.2.3/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -228,10 +211,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -239,6 +221,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -247,7 +231,6 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -280,8 +263,8 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -355,24 +338,20 @@ github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/ef-ds/deque v1.0.4 h1:iFAZNmveMT9WERAkqLJ+oaABF9AcVQ5AjXem/hroniI= github.com/ef-ds/deque v1.0.4/go.mod h1:gXDnTC3yqvBcHbq2lcExjtAcVrOnJCbMcZXmuj8Z4tg= -github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/ethereum/go-ethereum v1.10.1 h1:bGQezu+kqqRBczcSAruEoqVzTjtkeDnUGI2I4uroyUE= github.com/ethereum/go-ethereum v1.10.1/go.mod h1:E5e/zvdfUVr91JZ0AwjyuJM3x+no51zZJRz61orLLSk= @@ -396,7 +375,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fxamacker/cbor/v2 v2.2.1-0.20201006223149-25f67fca9803/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f h1:dxTR4AaxCwuQv9LAVTAC2r1szlS+epeuPT5ClLKT6ZY= github.com/fxamacker/cbor/v2 v2.4.1-0.20220515183430-ad2eae63303f/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= @@ -409,21 +387,21 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= -github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= -github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= -github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= -github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= +github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE= +github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= +github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= +github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= +github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -432,8 +410,6 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -455,7 +431,6 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= @@ -495,13 +470,11 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -515,7 +488,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -540,8 +512,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -554,7 +526,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -565,13 +536,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c h1:jk3Bnv5QzOSJil2OC6EQBoqy+lj/Kw56nrkEzvrQm2w= -github.com/google/pprof v0.0.0-20220818150347-1763105d910c/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811 h1:wORs2YN3R3ona/CXYuTvLM31QlgoNKHvlCNuArCDDCU= +github.com/google/pprof v0.0.0-20221219190121-3cb0bae90811/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -580,22 +546,14 @@ github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1 h1:kBRZU0PSuI7PspsSb/ChWoVResUcwNVIdpB049pKTiw= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -651,7 +609,6 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.0.0-20160813221303-0a025b7e63ad/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -667,7 +624,6 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20200620063722-49508fba0031/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= @@ -675,10 +631,11 @@ github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixH github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= @@ -836,8 +793,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -853,8 +808,8 @@ github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2vi github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= -github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kilic/bls12-381 v0.0.0-20201226121925-69dacb279461/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -863,15 +818,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= -github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= -github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= +github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -925,9 +880,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= -github.com/libp2p/go-libp2p v0.23.3 h1:/n3i0VtJF0iZ9YMUxl/teOY3h+M8NfgaCjOSYr9D+uI= -github.com/libp2p/go-libp2p v0.23.3/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= -github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= +github.com/libp2p/go-libp2p v0.24.2 h1:iMViPIcLY0D6zr/f+1Yq9EavCZu2i7eDstsr1nEwSAk= +github.com/libp2p/go-libp2p v0.24.2/go.mod h1:WuxtL2V8yGjam03D93ZBC19tvOUiPpewYv1xdFGWu1k= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -961,7 +915,6 @@ github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqe github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= -github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= @@ -974,10 +927,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.18.0 h1:akqO3gPMwixR7qFSFq70ezRun97g5hrA/lBW9jrjUYM= -github.com/libp2p/go-libp2p-kad-dht v0.18.0/go.mod h1:Gb92MYIPm3K2pJLGn8wl0m8wiKDvHrYpg+rOd0GzzPA= -github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= -github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= +github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -1001,7 +954,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRj github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= -github.com/libp2p/go-libp2p-peerstore v0.8.0 h1:bzTG693TA1Ju/zKmUCQzDLSqiJnyRFVwPpuloZ/OZtI= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-pubsub v0.8.2 h1:QLGUmkgKmwEVxVDYGsqc5t9CykOMY2Y21cXQHjR462I= github.com/libp2p/go-libp2p-pubsub v0.8.2/go.mod h1:e4kT+DYjzPUYGZeWk4I+oxCSYTXizzXii5LDRRhjKSw= @@ -1030,8 +982,6 @@ github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehts github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= -github.com/libp2p/go-libp2p-tls v0.4.1 h1:1ByJUbyoMXvYXDoW6lLsMxqMViQNXmt+CfQqlnCpY+M= -github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= @@ -1070,8 +1020,8 @@ github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdm github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= -github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= -github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= @@ -1115,12 +1065,11 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= -github.com/lucas-clemente/quic-go v0.29.1 h1:Z+WMJ++qMLhvpFkRZA+jl3BTxUjm415YBmWanXB8zP0= -github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= +github.com/lucas-clemente/quic-go v0.31.1 h1:O8Od7hfioqq0PMYHDyBkxU2aA7iZ2W9pjbrWuja2YR4= +github.com/lucas-clemente/quic-go v0.31.1/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -1133,25 +1082,23 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= +github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= +github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/marten-seemann/webtransport-go v0.1.1 h1:TnyKp3pEXcDooTaNn4s9dYpMJ7kMnTp7k5h+SgYP/mc= +github.com/marten-seemann/webtransport-go v0.4.3 h1:vkt5o/Ci+luknRteWdYGYH1KcB7ziup+J+1PzZJIvmg= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1162,8 +1109,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= @@ -1173,13 +1118,10 @@ github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnU github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1210,7 +1152,6 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -1223,7 +1164,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= @@ -1237,8 +1177,9 @@ github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -1250,8 +1191,8 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= -github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1273,8 +1214,8 @@ github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPw github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= -github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= -github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.7.0 h1:rTUjGOwjlhGHbEMbPoSUJowG1spZTVsITRANCjKTUAQ= +github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1296,8 +1237,9 @@ github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcR github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= @@ -1313,8 +1255,6 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -1323,25 +1263,20 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onflow/atree v0.4.0 h1:+TbNisavAkukAKhgQ4plWnvR9o5+SkwPIsi3jaeAqKs= github.com/onflow/atree v0.4.0/go.mod h1:7Qe1xaW0YewvouLXrugzMFUYXNoRQ8MT/UsVAWx1Ndo= -github.com/onflow/cadence v0.11.2/go.mod h1:8NwJGO535nnY/+QWEMDc2rhvOFChToWQ9Bg7fUIIc/I= -github.com/onflow/cadence v0.31.2 h1:13OTvfyACXyR3pv0Lkp2IAjXWMIXMee7Bn2yFz0onrM= -github.com/onflow/cadence v0.31.2/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722 h1:fH5e7L9xFXNOd3WLvMaPNkP1m7BngRTDP751zMNndws= -github.com/onflow/flow-core-contracts/lib/go/contracts v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:9nrgjIF/noY2jJ7LP8bKLHTpcdHOa2yO0ryCKTQpxvs= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722 h1:vgNS6I+MM/74pciIoKb7ZBs8XGF1ONsSdkAec36B9iU= -github.com/onflow/flow-core-contracts/lib/go/templates v0.11.2-0.20221216161720-c1b31d5a4722/go.mod h1:WMmeggH/H9Xb/SsT+4QFMtGYf+p1S2LXzbZAIaQQWAk= -github.com/onflow/flow-emulator v0.42.1-0.20230110232744-399decb21158 h1:53BnyMiQmMRipa3lWP0SLFGgMPybW75YI2L9Lv+C7Zc= -github.com/onflow/flow-emulator v0.42.1-0.20230110232744-399decb21158/go.mod h1:QDWRNuDs+yjKoq9ppj7kQOnoDonhX6Dizn/tcVVv/eE= +github.com/onflow/cadence v0.31.3 h1:lTHTBnoFi/ahGk1cQ9JAC+tfYir4kjRVIqtTVCjbe6E= +github.com/onflow/cadence v0.31.3/go.mod h1:oRgWkvau1RH15m3NuDlZCPHFQzwvC72jEstCGu8OJ98= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1 h1:9QEI+C9k/Cx/TRC3SCAHmNQqV7UlLG0DHQewTl8Lg6w= +github.com/onflow/flow-core-contracts/lib/go/contracts v0.12.1/go.mod h1:xiSs5IkirazpG89H5jH8xVUKNPlCZqUhLH4+vikQVS4= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1 h1:dhXSFiBkS6Q3XmBctJAfwR4XPkgBT7VNx08F/zTBgkM= +github.com/onflow/flow-core-contracts/lib/go/templates v0.12.1/go.mod h1:cBimYbTvHK77lclJ1JyhvmKAB9KDzCeWm7OW1EeQSr0= +github.com/onflow/flow-emulator v0.43.1-0.20230202181019-910459a16e2e h1:iKd4A+FOxjEpOBgMoVWepyt20bMZoxzPJ3FOggGpNjQ= +github.com/onflow/flow-emulator v0.43.1-0.20230202181019-910459a16e2e/go.mod h1:hC3NgLMbQRyxlTcv15NFdb/nZs7emi3yV9QDslxirQ4= github.com/onflow/flow-ft/lib/go/contracts v0.5.0 h1:Cg4gHGVblxcejfNNG5Mfj98Wf4zbY76O0Y28QB0766A= github.com/onflow/flow-ft/lib/go/contracts v0.5.0/go.mod h1:1zoTjp1KzNnOPkyqKmWKerUyf0gciw+e6tAEt0Ks3JE= -github.com/onflow/flow-ft/lib/go/templates v0.2.0 h1:oQQk5UthLS9KfKLkZVJg/XAVq8CXW7HAxSTu4HwBJkU= -github.com/onflow/flow-ft/lib/go/templates v0.2.0/go.mod h1:qwkTElMcI+PnSBGIWGu1K9OYBLatNimWTC8un9qUji0= -github.com/onflow/flow-go-sdk v0.13.0/go.mod h1:yqnSajzJVFfrTg68F4WXRR1Yzs1akqAjyscEDyFudPE= -github.com/onflow/flow-go-sdk v0.31.2 h1:DuTzsNjheaL5r9IQWtGC08Kq5JMZ4Uf2OA/Pst6bsSg= -github.com/onflow/flow-go-sdk v0.31.2/go.mod h1:l9Fgfg+aBx4ILvEE1Wf6g5G9Lg1A+hzY6oh4bQdkcFs= +github.com/onflow/flow-go-sdk v0.31.3 h1:CytMRiTayXRlkRNXQ9Cw6ZcKoOIK6+QtXk4iUb8f0zQ= +github.com/onflow/flow-go-sdk v0.31.3/go.mod h1:cqj2QShwC4DqxWzrg0+U7KxE2k7OJDGBxh8XZrJ4v5E= github.com/onflow/flow-go/crypto v0.24.4 h1:SwEtoVS2TidCIHYCZMgQ7U2YsqhI9upnw94fhdHTubM= github.com/onflow/flow-go/crypto v0.24.4/go.mod h1:dkVL98P6GHR48iD9zCB6XlnkJX8IQd00FKgt1reV90w= -github.com/onflow/flow/protobuf/go/flow v0.1.8/go.mod h1:kRugbzZjwQqvevJhrnnCFMJZNmoSJmxlKt6hTGXZojM= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288 h1:haWv3D5loiH+zcOoWEvDXtWQvXt5U8PLliQjwhv9sfw= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20221202093946-932d1c70e288/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= @@ -1355,8 +1290,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.6.1 h1:1xQPCjcqYw/J5LchOcp4/2q/jzJFjiAOc25chhnDw+Q= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1364,7 +1299,7 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= @@ -1410,6 +1345,8 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= +github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1418,7 +1355,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= -github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/plus3it/gorecurcopy v0.0.1 h1:H7AgvM0N/uIo7o1PQRlewEGQ92BNr7DqbPy5lnR3uJI= github.com/plus3it/gorecurcopy v0.0.1/go.mod h1:NvVTm4RX68A1vQbHmHunDO4OtBLVroT6CrsiqAzNyJA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -1437,8 +1373,6 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1458,10 +1392,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1472,24 +1404,20 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/psiemens/sconfig v0.1.0 h1:xfWqW+TRpih7mXZIqKYTmpRhlZLQ1kbxV8EjllPv76s= github.com/psiemens/sconfig v0.1.0/go.mod h1:+MLKqdledP/8G3rOBpknbLh0IclCf4WneJUtS26JB2U= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= -github.com/raviqqe/hamt v0.0.0-20190615202029-864fb7caef85/go.mod h1:I9elsTaXMhu41qARmzefHy7v2KmAV2TB1yH4E+nBSf0= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a h1:s7GrsqeorVkFR1vGmQ6WVL9nup0eyQCC+YVUeSQLH/Q= github.com/rivo/uniseg v0.2.1-0.20211004051800-57c86be7915a/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1501,8 +1429,8 @@ github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubr github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= -github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= -github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1511,7 +1439,6 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8= github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/segmentio/fasthash v1.0.2/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1553,6 +1480,10 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= +github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag= +github.com/slok/go-http-metrics v0.10.0 h1:rh0LaYEKza5eaYRGDXujKrOln57nHBi4TtVhmNEpbgM= +github.com/slok/go-http-metrics v0.10.0/go.mod h1:lFqdaS4kWMfUKCSukjC47PdCeTk+hXDUVm8kLHRqJ38= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -1566,7 +1497,6 @@ github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7A github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.0.1-0.20190317074736-539464a789e9/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -1579,8 +1509,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1603,8 +1533,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1614,8 +1545,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= @@ -1623,7 +1555,6 @@ github.com/supranational/blst v0.3.4 h1:iZE9lBMoywK2uy2U/5hDOvobQk9FnOQ2wNlu9GmR github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= @@ -1673,8 +1604,8 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+m github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= -github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= @@ -1688,6 +1619,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= @@ -1711,8 +1643,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 h1:ao8CJIShCaIbaMsGxy+jp2YHSudketpDgDRcbirov78= @@ -1735,6 +1668,10 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= +go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -1743,8 +1680,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1753,8 +1690,8 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1764,7 +1701,6 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1794,8 +1730,12 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1810,8 +1750,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4= -golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1827,7 +1767,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= @@ -1841,8 +1780,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1896,22 +1836,15 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1923,19 +1856,9 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1948,9 +1871,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1979,7 +1902,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1988,7 +1910,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2018,7 +1939,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2027,63 +1947,48 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2093,8 +1998,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2151,7 +2057,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -2161,32 +2066,26 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -2215,33 +2114,11 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.94.0 h1:KtKM9ru3nzQioV1HLlUf1cR7vMYJIpgls5VhAYQXIwA= -google.golang.org/api v0.94.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2289,61 +2166,16 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220902135211-223410557253 h1:vXJMM8Shg7TGaYxZsQ++A/FOSlbDmDtWhS/o+3w/hj4= -google.golang.org/genproto v0.0.0-20220902135211-223410557253/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2370,22 +2202,10 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -2401,7 +2221,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -2422,13 +2241,10 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= @@ -2445,6 +2261,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/integration/localnet/bootstrap.go b/integration/localnet/bootstrap.go index 628b2c63889..d42348fea69 100644 --- a/integration/localnet/bootstrap.go +++ b/integration/localnet/bootstrap.go @@ -348,7 +348,6 @@ func prepareConsensusService(container testnet.ContainerConfig, i int, n int) Se service.Command = append( service.Command, fmt.Sprintf("--block-rate-delay=%s", consensusDelay), - fmt.Sprintf("--hotstuff-timeout=%s", timeout), fmt.Sprintf("--hotstuff-min-timeout=%s", timeout), "--chunk-alpha=1", "--emergency-sealing-active=false", @@ -386,7 +385,6 @@ func prepareCollectionService(container testnet.ContainerConfig, i int, n int) S service.Command = append( service.Command, fmt.Sprintf("--block-rate-delay=%s", collectionDelay), - fmt.Sprintf("--hotstuff-timeout=%s", timeout), fmt.Sprintf("--hotstuff-min-timeout=%s", timeout), fmt.Sprintf("--ingress-addr=%s:%d", container.ContainerName, RPCPort), "--insecure-access-api=false", diff --git a/integration/testnet/client.go b/integration/testnet/client.go index 21b87878ba1..f46ddca5c11 100644 --- a/integration/testnet/client.go +++ b/integration/testnet/client.go @@ -193,6 +193,7 @@ func (c *Client) Account() *sdk.Account { return c.account } +// WaitForSealed waits for the transaction to be sealed, then returns the result. func (c *Client) WaitForSealed(ctx context.Context, id sdk.Identifier) (*sdk.TransactionResult, error) { fmt.Printf("Waiting for transaction %s to be sealed...\n", id) @@ -214,7 +215,7 @@ func (c *Client) WaitForSealed(ctx context.Context, id sdk.Identifier) (*sdk.Tra } else { fmt.Print(".") } - time.Sleep(time.Second) + time.Sleep(250 * time.Millisecond) } fmt.Println() @@ -223,7 +224,8 @@ func (c *Client) WaitForSealed(ctx context.Context, id sdk.Identifier) (*sdk.Tra return result, err } -// GetLatestProtocolSnapshot ... +// GetLatestProtocolSnapshot returns the latest protocol state snapshot. +// The snapshot head is latest finalized - tail of sealing segment is latest sealed. func (c *Client) GetLatestProtocolSnapshot(ctx context.Context) (*inmem.Snapshot, error) { b, err := c.client.GetLatestProtocolStateSnapshot(ctx) if err != nil { @@ -260,6 +262,16 @@ func (c *Client) GetLatestSealedBlockHeader(ctx context.Context) (*sdk.BlockHead return header, nil } +// GetLatestFinalizedBlockHeader returns full block header for the latest finalized block +func (c *Client) GetLatestFinalizedBlockHeader(ctx context.Context) (*sdk.BlockHeader, error) { + header, err := c.client.GetLatestBlockHeader(ctx, false) + if err != nil { + return nil, fmt.Errorf("could not get latest sealed block header: %w", err) + } + + return header, nil +} + func (c *Client) UserAddress(txResp *sdk.TransactionResult) (sdk.Address, bool) { var ( address sdk.Address @@ -280,6 +292,19 @@ func (c *Client) UserAddress(txResp *sdk.TransactionResult) (sdk.Address, bool) return address, found } +// CreatedAccounts returns the addresses of all accounts created in the given transaction, +// in the order they were created. +func (c *Client) CreatedAccounts(txResp *sdk.TransactionResult) []sdk.Address { + var addresses []sdk.Address + for _, event := range txResp.Events { + if event.Type == sdk.EventAccountCreated { + accountCreatedEvent := sdk.AccountCreatedEvent(event) + addresses = append(addresses, accountCreatedEvent.Address()) + } + } + return addresses +} + func (c *Client) TokenAmountByRole(role flow.Role) (string, float64, error) { if role == flow.RoleCollection { return "250000.0", 250000.0, nil @@ -294,7 +319,7 @@ func (c *Client) TokenAmountByRole(role flow.Role) (string, float64, error) { return "135000.0", 135000.0, nil } if role == flow.RoleAccess { - return "0.0", 0.0, nil + return "100.0", 100.0, nil } return "", 0, fmt.Errorf("could not get token amount by role: %v", role) diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 9d65f0eb9c2..0cd49c9681e 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -367,13 +367,14 @@ func NewConsensusFollowerConfig(t *testing.T, networkingPrivKey crypto.PrivateKe // NetworkConfig is the config for the network. type NetworkConfig struct { - Nodes []NodeConfig - ConsensusFollowers []ConsensusFollowerConfig - Name string - NClusters uint - ViewsInDKGPhase uint64 - ViewsInStakingAuction uint64 - ViewsInEpoch uint64 + Nodes []NodeConfig + ConsensusFollowers []ConsensusFollowerConfig + Name string + NClusters uint + ViewsInDKGPhase uint64 + ViewsInStakingAuction uint64 + ViewsInEpoch uint64 + EpochCommitSafetyThreshold uint64 } type NetworkConfigOpt func(*NetworkConfig) @@ -395,14 +396,15 @@ func NewNetworkConfig(name string, nodes []NodeConfig, opts ...NetworkConfigOpt) return c } -func NewNetworkConfigWithEpochConfig(name string, nodes []NodeConfig, viewsInStakingAuction, viewsInDKGPhase, viewsInEpoch uint64, opts ...NetworkConfigOpt) NetworkConfig { +func NewNetworkConfigWithEpochConfig(name string, nodes []NodeConfig, viewsInStakingAuction, viewsInDKGPhase, viewsInEpoch, safetyThreshold uint64, opts ...NetworkConfigOpt) NetworkConfig { c := NetworkConfig{ - Nodes: nodes, - Name: name, - NClusters: 1, // default to 1 cluster - ViewsInStakingAuction: viewsInStakingAuction, - ViewsInDKGPhase: viewsInDKGPhase, - ViewsInEpoch: viewsInEpoch, + Nodes: nodes, + Name: name, + NClusters: 1, // default to 1 cluster + ViewsInStakingAuction: viewsInStakingAuction, + ViewsInDKGPhase: viewsInDKGPhase, + ViewsInEpoch: viewsInEpoch, + EpochCommitSafetyThreshold: safetyThreshold, } for _, apply := range opts { @@ -430,6 +432,12 @@ func WithViewsInDKGPhase(views uint64) func(*NetworkConfig) { } } +func WithEpochCommitSafetyThreshold(threshold uint64) func(*NetworkConfig) { + return func(config *NetworkConfig) { + config.EpochCommitSafetyThreshold = threshold + } +} + func WithClusters(n uint) func(*NetworkConfig) { return func(conf *NetworkConfig) { conf.NClusters = n @@ -578,9 +586,15 @@ func WithAdditionalFlag(flag string) func(config *NodeConfig) { } } -// integrationBootstrapDir creates a temporary directory at /tmp/flow-integration-bootstrap -func integrationBootstrapDir() (string, error) { - return os.MkdirTemp(TmpRoot, integrationBootstrap) +// tempDir creates a temporary directory at /tmp/flow-integration-bootstrap +func tempDir(t *testing.T) string { + dir, err := os.MkdirTemp(TmpRoot, integrationBootstrap) + require.NoError(t, err) + t.Cleanup(func() { + err := os.RemoveAll(dir) + require.NoError(t, err) + }) + return dir } func PrepareFlowNetwork(t *testing.T, networkConf NetworkConfig, chainID flow.ChainID) *FlowNetwork { @@ -609,8 +623,7 @@ func PrepareFlowNetwork(t *testing.T, networkConf NetworkConfig, chainID flow.Ch }) // create a temporary directory to store all bootstrapping files - bootstrapDir, err := integrationBootstrapDir() - require.Nil(t, err) + bootstrapDir := tempDir(t) t.Logf("BootstrapDir: %s \n", bootstrapDir) @@ -813,7 +826,7 @@ func (net *FlowNetwork) AddObserver(t *testing.T, ctx context.Context, conf *Obs }() // Setup directories - tmpdir := t.TempDir() + tmpdir := tempDir(t) flowDataDir := net.makeDir(t, tmpdir, DefaultFlowDataDir) nodeBootstrapDir := net.makeDir(t, tmpdir, DefaultBootstrapDir) @@ -921,7 +934,7 @@ func (net *FlowNetwork) AddNode(t *testing.T, bootstrapDir string, nodeConf Cont HostConfig: &container.HostConfig{}, } - tmpdir := t.TempDir() + tmpdir := tempDir(t) t.Logf("%v adding container %v for %v node", time.Now().UTC(), nodeConf.ContainerName, nodeConf.Role) @@ -980,7 +993,6 @@ func (net *FlowNetwork) AddNode(t *testing.T, bootstrapDir string, nodeConf Cont // net.AccessPorts[ColNodeMetricsPort] = hostMetricsPort // net.MetricsPortsByContainerName[nodeContainer.Name()] = hostMetricsPort // set a low timeout so that all nodes agree on the current view more quickly - nodeContainer.AddFlag("hotstuff-timeout", time.Second.String()) nodeContainer.AddFlag("hotstuff-min-timeout", time.Second.String()) t.Logf("%v hotstuff startup time will be in 8 seconds: %v", time.Now().UTC(), hotstuffStartupTime) nodeContainer.AddFlag("hotstuff-startup-time", hotstuffStartupTime) @@ -1296,11 +1308,15 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl if err != nil { return nil, err } - qc, err := run.GenerateRootQC(root, votes, signerData, signerData.Identities()) + qc, invalidVotesErr, err := run.GenerateRootQC(root, votes, signerData, signerData.Identities()) if err != nil { return nil, err } + if len(invalidVotesErr) > 0 { + return nil, fmt.Errorf("has invalid votes: %v", invalidVotesErr) + } + // generate root blocks for each collector cluster clusterRootBlocks, clusterAssignments, clusterQCs, err := setupClusterGenesisBlockQCs(networkConf.NClusters, epochCounter, stakedConfs) if err != nil { @@ -1394,7 +1410,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl return nil, fmt.Errorf("generating root seal failed: %w", err) } - snapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) + snapshot, err := inmem.SnapshotFromBootstrapStateWithParams(root, result, seal, qc, flow.DefaultProtocolVersion, networkConf.EpochCommitSafetyThreshold) if err != nil { return nil, fmt.Errorf("could not create bootstrap state snapshot: %w", err) } diff --git a/integration/tests/access/consensus_follower_test.go b/integration/tests/access/consensus_follower_test.go index 43e26c7ecf5..e713d0c892c 100644 --- a/integration/tests/access/consensus_follower_test.go +++ b/integration/tests/access/consensus_follower_test.go @@ -132,7 +132,6 @@ func (suite *ConsensusFollowerSuite) buildNetworkConfig() { } consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), diff --git a/integration/tests/access/execution_state_sync_test.go b/integration/tests/access/execution_state_sync_test.go index c58fb1561eb..f75328776a2 100644 --- a/integration/tests/access/execution_state_sync_test.go +++ b/integration/tests/access/execution_state_sync_test.go @@ -93,7 +93,6 @@ func (s *ExecutionStateSyncSuite) buildNetworkConfig() { testnet.AsGhost()) consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), diff --git a/admin/command_runner_test.go b/integration/tests/admin/command_runner_test.go similarity index 91% rename from admin/command_runner_test.go rename to integration/tests/admin/command_runner_test.go index a025a6ae6df..9a354632d89 100644 --- a/admin/command_runner_test.go +++ b/integration/tests/admin/command_runner_test.go @@ -30,6 +30,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" + "github.com/onflow/flow-go/admin" pb "github.com/onflow/flow-go/admin/admin" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/utils/grpcutils" @@ -39,8 +40,8 @@ import ( type CommandRunnerSuite struct { suite.Suite - runner *CommandRunner - bootstrapper *CommandRunnerBootstrapper + runner *admin.CommandRunner + bootstrapper *admin.CommandRunnerBootstrapper httpAddress string grpcAddressSock string @@ -56,7 +57,7 @@ func TestCommandRunner(t *testing.T) { func (suite *CommandRunnerSuite) SetupTest() { suite.httpAddress = unittest.IPPort(testingdock.RandomPort(suite.T())) - suite.bootstrapper = NewCommandRunnerBootstrapper() + suite.bootstrapper = admin.NewCommandRunnerBootstrapper() } func (suite *CommandRunnerSuite) TearDownTest() { @@ -72,21 +73,21 @@ func (suite *CommandRunnerSuite) TearDownTest() { <-suite.runner.Done() } -func (suite *CommandRunnerSuite) SetupCommandRunner(opts ...CommandRunnerOption) { +func (suite *CommandRunnerSuite) SetupCommandRunner(opts ...admin.CommandRunnerOption) { ctx, cancel := context.WithCancel(context.Background()) suite.cancel = cancel signalerCtx := irrecoverable.NewMockSignalerContext(suite.T(), ctx) suite.grpcAddressSock = fmt.Sprintf("%s/%s-flow-node-admin.sock", os.TempDir(), unittest.GenerateRandomStringWithLen(16)) - opts = append(opts, WithGRPCAddress(suite.grpcAddressSock), WithMaxMsgSize(grpcutils.DefaultMaxMsgSize)) + opts = append(opts, admin.WithGRPCAddress(suite.grpcAddressSock), admin.WithMaxMsgSize(grpcutils.DefaultMaxMsgSize)) logger := zerolog.New(zerolog.NewConsoleWriter()) suite.runner = suite.bootstrapper.Bootstrap(logger, suite.httpAddress, opts...) suite.runner.Start(signalerCtx) <-suite.runner.Ready() - conn, err := grpc.Dial("unix:///"+suite.runner.grpcAddress, grpc.WithTransportCredentials(grpcinsecure.NewCredentials())) + conn, err := grpc.Dial("unix:///"+suite.runner.GrpcAddress(), grpc.WithTransportCredentials(grpcinsecure.NewCredentials())) suite.NoError(err) suite.conn = conn suite.client = pb.NewAdminClient(conn) @@ -95,7 +96,7 @@ func (suite *CommandRunnerSuite) SetupCommandRunner(opts ...CommandRunnerOption) func (suite *CommandRunnerSuite) TestHandler() { called := false - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { select { case <-ctx.Done(): return nil, ctx.Err() @@ -153,7 +154,7 @@ func (suite *CommandRunnerSuite) TestUnimplementedHandler() { func (suite *CommandRunnerSuite) TestValidator() { calls := 0 - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { select { case <-ctx.Done(): return nil, ctx.Err() @@ -165,8 +166,8 @@ func (suite *CommandRunnerSuite) TestValidator() { return "ok", nil }) - validatorErr := NewInvalidAdminReqErrorf("unexpected value") - suite.bootstrapper.RegisterValidator("foo", func(req *CommandRequest) error { + validatorErr := admin.NewInvalidAdminReqErrorf("unexpected value") + suite.bootstrapper.RegisterValidator("foo", func(req *admin.CommandRequest) error { if req.Data.(map[string]interface{})["key"] != "value" { return validatorErr } @@ -203,7 +204,7 @@ func (suite *CommandRunnerSuite) TestValidator() { func (suite *CommandRunnerSuite) TestHandlerError() { handlerErr := errors.New("handler error") - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { select { case <-ctx.Done(): return nil, ctx.Err() @@ -233,7 +234,7 @@ func (suite *CommandRunnerSuite) TestHandlerError() { } func (suite *CommandRunnerSuite) TestTimeout() { - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { <-ctx.Done() return nil, ctx.Err() }) @@ -259,7 +260,7 @@ func (suite *CommandRunnerSuite) TestTimeout() { func (suite *CommandRunnerSuite) TestHTTPServer() { called := false - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { select { case <-ctx.Done(): return nil, ctx.Err() @@ -305,13 +306,13 @@ func (suite *CommandRunnerSuite) TestHTTPPProf() { } func (suite *CommandRunnerSuite) TestListCommands() { - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { return nil, nil }) - suite.bootstrapper.RegisterHandler("bar", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("bar", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { return nil, nil }) - suite.bootstrapper.RegisterHandler("baz", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("baz", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { return nil, nil }) @@ -444,7 +445,7 @@ func generateCerts(t *testing.T) (tls.Certificate, *x509.CertPool, tls.Certifica func (suite *CommandRunnerSuite) TestTLS() { called := false - suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *CommandRequest) (interface{}, error) { + suite.bootstrapper.RegisterHandler("foo", func(ctx context.Context, req *admin.CommandRequest) (interface{}, error) { select { case <-ctx.Done(): return nil, ctx.Err() @@ -470,7 +471,7 @@ func (suite *CommandRunnerSuite) TestTLS() { RootCAs: serverCertPool, } - suite.SetupCommandRunner(WithTLS(serverConfig)) + suite.SetupCommandRunner(admin.WithTLS(serverConfig)) client := &http.Client{ Transport: &http.Transport{ diff --git a/integration/tests/bft/passthrough/dummy_orchestrator.go b/integration/tests/bft/passthrough/dummy_orchestrator.go index abc631e0a5c..cae9713f324 100644 --- a/integration/tests/bft/passthrough/dummy_orchestrator.go +++ b/integration/tests/bft/passthrough/dummy_orchestrator.go @@ -1,7 +1,9 @@ package passthrough import ( + "errors" "fmt" + "io" "sync" "testing" @@ -103,6 +105,11 @@ func (d *dummyOrchestrator) HandleIngressEvent(event *insecure.IngressEvent) err err := d.orchestratorNetwork.SendIngress(event) if err != nil { + if errors.Is(err, io.EOF) { + // log a warning and continue for EOF errors + lg.Err(err).Msg("could not pass through ingress event") + return nil + } // since this is used for testing, if we encounter any RPC send error, crash the orchestrator. lg.Error().Err(err).Msg("could not pass through ingress event") return err diff --git a/integration/tests/collection/suite.go b/integration/tests/collection/suite.go index f1b8f3fe11b..8245cf92b8f 100644 --- a/integration/tests/collection/suite.go +++ b/integration/tests/collection/suite.go @@ -351,7 +351,8 @@ func (suite *CollectorSuite) ClusterStateFor(id flow.Identifier) *clusterstateim db, err := node.DB() require.Nil(suite.T(), err, "could not get node db") - clusterStateRoot, err := clusterstateimpl.NewStateRoot(rootBlock) + rootQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(rootBlock.ID())) + clusterStateRoot, err := clusterstateimpl.NewStateRoot(rootBlock, rootQC) suite.NoError(err) clusterState, err := clusterstateimpl.OpenState(db, nil, nil, nil, clusterStateRoot.ClusterID()) require.NoError(suite.T(), err, "could not get cluster state") diff --git a/integration/tests/epochs/epoch_join_and_leave_an_test.go b/integration/tests/epochs/epoch_join_and_leave_an_test.go index 8387b1b939c..25b96bf425a 100644 --- a/integration/tests/epochs/epoch_join_and_leave_an_test.go +++ b/integration/tests/epochs/epoch_join_and_leave_an_test.go @@ -6,11 +6,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" ) func TestEpochJoinAndLeaveAN(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "epochs join/leave tests should be run on an machine with adequate resources") suite.Run(t, new(EpochJoinAndLeaveANSuite)) } diff --git a/integration/tests/epochs/epoch_join_and_leave_sn_test.go b/integration/tests/epochs/epoch_join_and_leave_sn_test.go index 61dbb655c65..a3763420cdc 100644 --- a/integration/tests/epochs/epoch_join_and_leave_sn_test.go +++ b/integration/tests/epochs/epoch_join_and_leave_sn_test.go @@ -6,11 +6,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" ) func TestEpochJoinAndLeaveSN(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "epochs join/leave tests should be run on an machine with adequate resources") suite.Run(t, new(EpochJoinAndLeaveSNSuite)) } diff --git a/integration/tests/epochs/epoch_join_and_leave_vn_test.go b/integration/tests/epochs/epoch_join_and_leave_vn_test.go index 1aedb3d3356..f5ea2b09de0 100644 --- a/integration/tests/epochs/epoch_join_and_leave_vn_test.go +++ b/integration/tests/epochs/epoch_join_and_leave_vn_test.go @@ -6,11 +6,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" ) func TestEpochJoinAndLeaveVN(t *testing.T) { - unittest.SkipUnless(t, unittest.TEST_FLAKY, "epochs join/leave tests should be run on an machine with adequate resources") suite.Run(t, new(EpochJoinAndLeaveVNSuite)) } @@ -18,6 +16,20 @@ type EpochJoinAndLeaveVNSuite struct { DynamicEpochTransitionSuite } +func (s *EpochJoinAndLeaveVNSuite) SetupTest() { + // require approvals for seals to verify that the joining VN is producing valid seals in the second epoch + s.RequiredSealApprovals = 1 + // increase epoch length to account for greater sealing lag due to above + // NOTE: this value is set fairly aggressively to ensure shorter test times. + // If flakiness due to failure to complete staking operations in time is observed, + // try increasing (by 10-20 views). + s.StakingAuctionLen = 100 + s.DKGPhaseLen = 100 + s.EpochLen = 450 + s.EpochCommitSafetyThreshold = 20 + s.DynamicEpochTransitionSuite.SetupTest() +} + // TestEpochJoinAndLeaveVN should update verification nodes and assert healthy network conditions // after the epoch transition completes. See health check function for details. func (s *EpochJoinAndLeaveVNSuite) TestEpochJoinAndLeaveVN() { diff --git a/integration/tests/epochs/epoch_static_transition_test.go b/integration/tests/epochs/epoch_static_transition_test.go index 1b54d4b0f5e..d8ede87166f 100644 --- a/integration/tests/epochs/epoch_static_transition_test.go +++ b/integration/tests/epochs/epoch_static_transition_test.go @@ -2,6 +2,7 @@ package epochs import ( "testing" + "time" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -24,7 +25,8 @@ func (s *StaticEpochTransitionSuite) SetupTest() { // staking phase for this test s.StakingAuctionLen = 10 s.DKGPhaseLen = 50 - s.EpochLen = 200 + s.EpochLen = 300 + s.EpochCommitSafetyThreshold = 50 // run the generic setup, which starts up the network s.Suite.SetupTest() @@ -38,7 +40,7 @@ func (s *StaticEpochTransitionSuite) SetupTest() { func (s *StaticEpochTransitionSuite) TestStaticEpochTransition() { s.TimedLogf("waiting for EpochSetup phase of first epoch to begin") - s.WaitForPhase(s.ctx, flow.EpochPhaseSetup) + s.AwaitEpochPhase(s.ctx, 0, flow.EpochPhaseSetup, time.Minute, 500*time.Millisecond) s.TimedLogf("successfully reached EpochSetup phase of first epoch") snapshot, err := s.client.GetLatestProtocolSnapshot(s.ctx) @@ -53,14 +55,14 @@ func (s *StaticEpochTransitionSuite) TestStaticEpochTransition() { epoch1Counter, err := snapshot.Epochs().Current().Counter() require.NoError(s.T(), err) - // wait for the final view of the first epoch - s.TimedLogf("waiting for the final view (%d) of epoch %d", epoch1FinalView, epoch1Counter) - s.BlockState.WaitForSealedView(s.T(), epoch1FinalView+5) - s.TimedLogf("sealed final view (%d) of epoch %d", epoch1FinalView, epoch1Counter) + // wait for the first view of the second epoch + s.TimedLogf("waiting for the first view (%d) of second epoch %d", epoch1FinalView+1, epoch1Counter+1) + s.AwaitFinalizedView(s.ctx, epoch1FinalView+1, 4*time.Minute, 500*time.Millisecond) + s.TimedLogf("finalized first view (%d) of second epoch %d", epoch1FinalView+1, epoch1Counter+1) // assert transition to second epoch happened as expected // if counter is still 0, epoch emergency fallback was triggered and we can fail early - s.assertEpochCounter(s.ctx, 1) + s.AssertInEpoch(s.ctx, epoch1Counter+1) // submit a smoke test transaction to verify the network can seal a transaction s.TimedLogf("sending smoke test transaction in second epoch") diff --git a/integration/tests/epochs/suite.go b/integration/tests/epochs/suite.go index 42ea68cea92..56ddfe642c7 100644 --- a/integration/tests/epochs/suite.go +++ b/integration/tests/epochs/suite.go @@ -14,6 +14,8 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/fvm/blueprints" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/ghost/client" @@ -27,36 +29,39 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -const waitTimeout = 2 * time.Minute - // nodeUpdateValidation func that will be used to validate the health of the network // after an identity table change during an epoch transition. This is used in // tandem with runTestEpochJoinAndLeave. -// -// NOTE: rootSnapshot must be the snapshot that the node (info) was bootstrapped with. -type nodeUpdateValidation func(ctx context.Context, env templates.Environment, rootSnapshot *inmem.Snapshot, info *StakedNodeOperationInfo) +// NOTE: The snapshot must reference a block within the second epoch. +type nodeUpdateValidation func(ctx context.Context, env templates.Environment, snapshot *inmem.Snapshot, info *StakedNodeOperationInfo) // Suite encapsulates common functionality for epoch integration tests. type Suite struct { suite.Suite - log zerolog.Logger lib.TestnetStateTracker ctx context.Context cancel context.CancelFunc + log zerolog.Logger net *testnet.FlowNetwork ghostID flow.Identifier client *testnet.Client // Epoch config (lengths in views) - StakingAuctionLen uint64 - DKGPhaseLen uint64 - EpochLen uint64 + StakingAuctionLen uint64 + DKGPhaseLen uint64 + EpochLen uint64 + EpochCommitSafetyThreshold uint64 + // Whether approvals are required for sealing (we only enable for VN tests because + // requiring approvals requires a longer DKG period to avoid flakiness) + RequiredSealApprovals uint // defaults to 0 (no approvals required) } // SetupTest is run automatically by the testing framework before each test case. func (s *Suite) SetupTest() { + + minEpochLength := s.StakingAuctionLen + s.DKGPhaseLen*3 + 20 // ensure epoch lengths are set correctly - require.Greater(s.T(), s.EpochLen, s.StakingAuctionLen+s.DKGPhaseLen*3) + require.Greater(s.T(), s.EpochLen, minEpochLength+s.EpochCommitSafetyThreshold, "epoch too short") s.ctx, s.cancel = context.WithCancel(context.Background()) s.log = unittest.LoggerForTest(s.Suite.T(), zerolog.InfoLevel) @@ -66,18 +71,14 @@ func (s *Suite) SetupTest() { }() collectionConfigs := []func(*testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), - testnet.WithLogLevel(zerolog.WarnLevel), - } + testnet.WithLogLevel(zerolog.WarnLevel)} consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), - testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), - testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), - testnet.WithLogLevel(zerolog.WarnLevel), - } + testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", s.RequiredSealApprovals)), + testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", s.RequiredSealApprovals)), + testnet.WithLogLevel(zerolog.WarnLevel)} // a ghost node masquerading as an access node s.ghostID = unittest.IdentifierFixture() @@ -88,33 +89,39 @@ func (s *Suite) SetupTest() { testnet.AsGhost()) confs := []testnet.NodeConfig{ + testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)), + testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)), testnet.NewNodeConfig(flow.RoleCollection, collectionConfigs...), - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel), testnet.WithAdditionalFlag("--extensive-logging=true")), - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel)), testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel), testnet.WithAdditionalFlag("--extensive-logging=true")), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.WarnLevel)), testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.WarnLevel)), - testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.WarnLevel)), ghostNode, } - netConf := testnet.NewNetworkConfigWithEpochConfig("epochs-tests", confs, s.StakingAuctionLen, s.DKGPhaseLen, s.EpochLen) + netConf := testnet.NewNetworkConfigWithEpochConfig("epochs-tests", confs, s.StakingAuctionLen, s.DKGPhaseLen, s.EpochLen, s.EpochCommitSafetyThreshold) // initialize the network s.net = testnet.PrepareFlowNetwork(s.T(), netConf, flow.Localnet) // start the network - s.net.Start(s.ctx) // start tracking blocks s.Track(s.T(), s.ctx, s.Ghost()) - addr := fmt.Sprintf(":%s", s.net.AccessPorts[testnet.AccessNodeAPIPort]) + // use AN1 for test-related queries - the AN join/leave test will replace AN2 + port, ok := s.net.AccessPortsByContainerName["access_1"] + require.True(s.T(), ok) + addr := fmt.Sprintf(":%s", port) client, err := testnet.NewClient(addr, s.net.Root().Header.ChainID.Chain()) require.NoError(s.T(), err) s.client = client + + // log network info periodically to aid in debugging future flaky tests + go lib.LogStatusPeriodically(s.T(), s.ctx, s.log, s.client, 5*time.Second) } func (s *Suite) Ghost() *client.GhostClient { @@ -124,8 +131,10 @@ func (s *Suite) Ghost() *client.GhostClient { return client } -// TimedLogf logs the message using t.Log, but prefixes the current time. +// TimedLogf logs the message using t.Log and the suite logger, but prefixes the current time. +// This enables viewing logs inline with Docker logs as well as other test logs. func (s *Suite) TimedLogf(msg string, args ...interface{}) { + s.log.Info().Msgf(msg, args...) args = append([]interface{}{time.Now().String()}, args...) s.T().Logf("%s - "+msg, args...) } @@ -161,71 +170,70 @@ type StakedNodeOperationInfo struct { // 4. Add additional funds to staking account for storage // 5. Create Staking collection for node // 6. Register node using staking collection object +// 7. Add the node to the approved list +// +// NOTE: assumes staking occurs in first epoch (counter 0) +// NOTE 2: This function performs steps 1-6 in one custom transaction, to reduce +// the time taken by each test case. Individual transactions for each step can be +// found in Git history, for example: 9867056a8b7246655047bc457f9000398f6687c0. func (s *Suite) StakeNode(ctx context.Context, env templates.Environment, role flow.Role) *StakedNodeOperationInfo { stakingAccountKey, networkingKey, stakingKey, machineAccountKey, machineAccountPubKey := s.generateAccountKeys(role) nodeID := flow.MakeID(stakingKey.PublicKey().Encode()) - fullAccountKey := sdk.NewAccountKey(). + fullStakingAcctKey := sdk.NewAccountKey(). SetPublicKey(stakingAccountKey.PublicKey()). SetHashAlgo(sdkcrypto.SHA2_256). SetWeight(sdk.AccountKeyWeightThreshold) - // create staking account - stakingAccountAddress, err := s.createAccount( - ctx, - fullAccountKey, - s.client.Account(), - s.client.SDKServiceAddress(), - ) - require.NoError(s.T(), err) - _, stakeAmount, err := s.client.TokenAmountByRole(role) require.NoError(s.T(), err) - // fund account with token amount to stake - result, err := s.fundAccount(ctx, stakingAccountAddress, fmt.Sprintf("%f", stakeAmount+10.0)) - require.NoError(s.T(), err) - require.NoError(s.T(), result.Error) - - stakingAccount, err := s.client.GetAccount(stakingAccountAddress) - require.NoError(s.T(), err) + containerName := s.getTestContainerName(role) - // create staking collection - result, err = s.createStakingCollection(ctx, env, stakingAccountKey, stakingAccount) + latestBlockID, err := s.client.GetLatestBlockID(ctx) require.NoError(s.T(), err) - require.NoError(s.T(), result.Error) - - containerName := s.getTestContainerName(role) - // register node using staking collection - result, machineAccountAddr, err := s.SubmitStakingCollectionRegisterNodeTx( - ctx, + // create and register node + tx, err := utils.MakeCreateAndSetupNodeTx( env, - stakingAccountKey, - stakingAccount, + s.client.Account(), + sdk.Identifier(latestBlockID), + fullStakingAcctKey, + fmt.Sprintf("%f", stakeAmount+10.0), nodeID, role, testnet.GetPrivateNodeInfoAddress(containerName), strings.TrimPrefix(networkingKey.PublicKey().String(), "0x"), strings.TrimPrefix(stakingKey.PublicKey().String(), "0x"), - fmt.Sprintf("%f", stakeAmount), machineAccountPubKey, ) + require.NoError(s.T(), err) + err = s.client.SignAndSendTransaction(ctx, tx) + require.NoError(s.T(), err) + result, err := s.client.WaitForSealed(ctx, tx.ID()) require.NoError(s.T(), err) + s.client.Account().Keys[0].SequenceNumber++ require.NoError(s.T(), result.Error) + accounts := s.client.CreatedAccounts(result) + stakingAccountAddress := accounts[0] + var machineAccountAddr sdk.Address + if role == flow.RoleCollection || role == flow.RoleConsensus { + machineAccountAddr = accounts[1] + } + result = s.SubmitSetApprovedListTx(ctx, env, append(s.net.Identities().NodeIDs(), nodeID)...) require.NoError(s.T(), result.Error) // ensure we are still in staking auction - s.assertInPhase(ctx, flow.EpochPhaseStaking) + s.AssertInEpochPhase(ctx, 0, flow.EpochPhaseStaking) return &StakedNodeOperationInfo{ NodeID: nodeID, Role: role, StakingAccountAddress: stakingAccountAddress, - FullAccountKey: fullAccountKey, + FullAccountKey: fullStakingAcctKey, StakingAccountKey: stakingAccountKey, StakingKey: stakingKey, NetworkingKey: networkingKey, @@ -236,50 +244,6 @@ func (s *Suite) StakeNode(ctx context.Context, env templates.Environment, role f } } -// WaitForPhase waits for epoch phase and will timeout after 2 minutes -func (s *Suite) WaitForPhase(ctx context.Context, phase flow.EpochPhase) { - condition := func() bool { - snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) - require.NoError(s.T(), err) - - currentPhase, err := snapshot.Phase() - require.NoError(s.T(), err) - - return currentPhase == phase - } - require.Eventually(s.T(), - condition, - waitTimeout, - 100*time.Millisecond, - fmt.Sprintf("did not reach epoch phase (%s) within %v seconds", phase, waitTimeout)) -} - -// transfers tokens to receiver from service account -func (s *Suite) fundAccount(ctx context.Context, receiver sdk.Address, tokenAmount string) (*sdk.TransactionResult, error) { - latestBlockID, err := s.client.GetLatestBlockID(ctx) - require.NoError(s.T(), err) - - env := utils.LocalnetEnv() - transferTx, err := utils.MakeTransferTokenTx( - env, - receiver, - s.client.Account(), - 0, - tokenAmount, - sdk.Identifier(latestBlockID), - ) - require.NoError(s.T(), err) - - err = s.client.SignAndSendTransaction(ctx, transferTx) - require.NoError(s.T(), err) - - result, err := s.client.WaitForSealed(ctx, transferTx.ID()) - require.NoError(s.T(), err) - s.client.Account().Keys[0].SequenceNumber++ - - return result, nil -} - // generates initial keys needed to bootstrap account func (s *Suite) generateAccountKeys(role flow.Role) ( operatorAccountKey, @@ -307,7 +271,7 @@ func (s *Suite) generateAccountKeys(role flow.Role) ( return } -// creates a new flow account, can be used to test staking +// createAccount creates a new flow account, can be used to test staking func (s *Suite) createAccount(ctx context.Context, accountKey *sdk.AccountKey, payerAccount *sdk.Account, @@ -323,136 +287,15 @@ func (s *Suite) createAccount(ctx context.Context, return addr, nil } -// creates a staking collection for the given node -func (s *Suite) createStakingCollection(ctx context.Context, env templates.Environment, accountKey sdkcrypto.PrivateKey, stakingAccount *sdk.Account) (*sdk.TransactionResult, error) { - latestBlockID, err := s.client.GetLatestBlockID(ctx) - require.NoError(s.T(), err) - - signer, err := sdkcrypto.NewInMemorySigner(accountKey, sdkcrypto.SHA2_256) - require.NoError(s.T(), err) - - createStakingCollectionTx, err := utils.MakeCreateStakingCollectionTx( - env, - stakingAccount, - 0, - signer, - s.client.SDKServiceAddress(), - sdk.Identifier(latestBlockID), - ) - require.NoError(s.T(), err) - - err = s.client.SignAndSendTransaction(ctx, createStakingCollectionTx) - require.NoError(s.T(), err) - - result, err := s.client.WaitForSealed(ctx, createStakingCollectionTx.ID()) - require.NoError(s.T(), err) - stakingAccount.Keys[0].SequenceNumber++ - - return result, nil -} - -// SubmitStakingCollectionRegisterNodeTx submits tx that calls StakingCollection.registerNode -func (s *Suite) SubmitStakingCollectionRegisterNodeTx( - ctx context.Context, - env templates.Environment, - accountKey sdkcrypto.PrivateKey, - stakingAccount *sdk.Account, - nodeID flow.Identifier, - role flow.Role, - networkingAddress string, - networkingKey string, - stakingKey string, - amount string, - machineKey *sdk.AccountKey, -) (*sdk.TransactionResult, sdk.Address, error) { - latestBlockID, err := s.client.GetLatestBlockID(ctx) - require.NoError(s.T(), err) - - signer, err := sdkcrypto.NewInMemorySigner(accountKey, sdkcrypto.SHA2_256) - require.NoError(s.T(), err) - - registerNodeTx, err := utils.MakeStakingCollectionRegisterNodeTx( - env, - stakingAccount, - 0, - signer, - s.client.SDKServiceAddress(), - sdk.Identifier(latestBlockID), - nodeID, - role, - networkingAddress, - networkingKey, - stakingKey, - amount, - machineKey, - ) - require.NoError(s.T(), err) - - err = s.client.SignAndSendTransaction(ctx, registerNodeTx) - require.NoError(s.T(), err) - - result, err := s.client.WaitForSealed(ctx, registerNodeTx.ID()) - require.NoError(s.T(), err) - stakingAccount.Keys[0].SequenceNumber++ - - if role == flow.RoleCollection || role == flow.RoleConsensus { - var machineAccountAddr sdk.Address - for _, event := range result.Events { - if event.Type == sdk.EventAccountCreated { // assume only one account created (safe because we control the transaction) - accountCreatedEvent := sdk.AccountCreatedEvent(event) - machineAccountAddr = accountCreatedEvent.Address() - break - } - } - - require.NotZerof(s.T(), machineAccountAddr, "failed to create the machine account: %s", machineAccountAddr) - return result, machineAccountAddr, nil - } - - return result, sdk.Address{}, nil -} - -// SubmitStakingCollectionCloseStakeTx submits tx that calls StakingCollection.closeStake -func (s *Suite) SubmitStakingCollectionCloseStakeTx( - ctx context.Context, - env templates.Environment, - accountKey sdkcrypto.PrivateKey, - stakingAccount *sdk.Account, - nodeID flow.Identifier, -) (*sdk.TransactionResult, error) { - latestBlockID, err := s.client.GetLatestBlockID(ctx) - require.NoError(s.T(), err) - - signer, err := sdkcrypto.NewInMemorySigner(accountKey, sdkcrypto.SHA2_256) - require.NoError(s.T(), err) - - closeStakeTx, err := utils.MakeStakingCollectionCloseStakeTx( - env, - stakingAccount, - 0, - signer, - s.client.SDKServiceAddress(), - sdk.Identifier(latestBlockID), - nodeID, - ) - require.NoError(s.T(), err) - - err = s.client.SignAndSendTransaction(ctx, closeStakeTx) - require.NoError(s.T(), err) - - result, err := s.client.WaitForSealed(ctx, closeStakeTx.ID()) - require.NoError(s.T(), err) - stakingAccount.Keys[0].SequenceNumber++ - return result, nil -} - +// removeNodeFromProtocol removes the given node from the protocol. +// NOTE: assumes staking occurs in first epoch (counter 0) func (s *Suite) removeNodeFromProtocol(ctx context.Context, env templates.Environment, nodeID flow.Identifier) { result, err := s.submitAdminRemoveNodeTx(ctx, env, nodeID) require.NoError(s.T(), err) require.NoError(s.T(), result.Error) // ensure we submit transaction while in staking phase - s.assertInPhase(ctx, flow.EpochPhaseStaking) + s.AssertInEpochPhase(ctx, 0, flow.EpochPhaseStaking) } // submitAdminRemoveNodeTx will submit the admin remove node transaction @@ -487,16 +330,17 @@ func (s *Suite) ExecuteGetProposedTableScript(ctx context.Context, env templates return v } -// SubmitSetApprovedListTx adds a node the the approved node list, this must be done when a node joins the protocol during the epoch staking phase -func (s *Suite) SubmitSetApprovedListTx(ctx context.Context, env templates.Environment, identities ...flow.Identifier) *sdk.TransactionResult { - ids := make([]cadence.Value, 0) - for _, id := range identities { - idCDC, err := cadence.NewString(id.String()) - require.NoError(s.T(), err) - - ids = append(ids, idCDC) - } +// ExecuteGetNodeInfoScript executes a script to get staking info about the given node. +func (s *Suite) ExecuteGetNodeInfoScript(ctx context.Context, env templates.Environment, nodeID flow.Identifier) cadence.Value { + cdcNodeID, err := cadence.NewString(nodeID.String()) + require.NoError(s.T(), err) + v, err := s.client.ExecuteScriptBytes(ctx, templates.GenerateGetNodeInfoScript(env), []cadence.Value{cdcNodeID}) + require.NoError(s.T(), err) + return v +} +// SubmitSetApprovedListTx adds a node to the approved node list, this must be done when a node joins the protocol during the epoch staking phase +func (s *Suite) SubmitSetApprovedListTx(ctx context.Context, env templates.Environment, identities ...flow.Identifier) *sdk.TransactionResult { latestBlockID, err := s.client.GetLatestBlockID(ctx) require.NoError(s.T(), err) @@ -508,7 +352,7 @@ func (s *Suite) SubmitSetApprovedListTx(ctx context.Context, env templates.Envir SetProposalKey(s.client.SDKServiceAddress(), 0, s.client.Account().Keys[0].SequenceNumber). SetPayer(s.client.SDKServiceAddress()). AddAuthorizer(idTableAddress) - err = tx.AddArgument(cadence.NewArray(ids)) + err = tx.AddArgument(blueprints.SetStakingAllowlistTxArg(identities)) require.NoError(s.T(), err) err = s.client.SignAndSendTransaction(ctx, tx) @@ -547,18 +391,6 @@ func (s *Suite) assertNodeApprovedAndProposed(ctx context.Context, env templates require.Containsf(s.T(), proposedTable.(cadence.Array).Values, cadence.String(info.NodeID.String()), "expected new node to be in proposed table: %x", info.NodeID) } -// assertNodeNotApprovedOrProposed executes the read approved nodes list and get proposed table scripts -// and checks that the info.NodeID is not included in either - this means the node would be excluded from future epochs -func (s *Suite) assertNodeNotApprovedOrProposed(ctx context.Context, env templates.Environment, nodeID flow.Identifier) { - // ensure node ID not in approved list - approvedNodes := s.ExecuteReadApprovedNodesScript(ctx, env) - require.NotContainsf(s.T(), approvedNodes.(cadence.Array).Values, cadence.String(nodeID.String()), "expected new node to not be in approved nodes list: %x", nodeID) - - // check if node is not in proposed table - proposedTable := s.ExecuteGetProposedTableScript(ctx, env, nodeID) - require.NotContainsf(s.T(), proposedTable.(cadence.Array).Values, cadence.String(nodeID.String()), "expected new node to not be in proposed table: %x", nodeID) -} - // newTestContainerOnNetwork configures a new container on the suites network func (s *Suite) newTestContainerOnNetwork(role flow.Role, info *StakedNodeOperationInfo) *testnet.Container { containerConfigs := []func(config *testnet.NodeConfig){ @@ -621,35 +453,97 @@ func (s *Suite) getContainerToReplace(role flow.Role) *testnet.Container { return nil } -// assertInPhase checks if we are in the phase provided and returns the current view -func (s *Suite) assertInPhase(ctx context.Context, expectedPhase flow.EpochPhase) { +// AwaitEpochPhase waits for the given phase, in the given epoch. +func (s *Suite) AwaitEpochPhase(ctx context.Context, expectedEpoch uint64, expectedPhase flow.EpochPhase, waitFor, tick time.Duration) { + condition := func() bool { + snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) + require.NoError(s.T(), err) + + actualEpoch, err := snapshot.Epochs().Current().Counter() + require.NoError(s.T(), err) + actualPhase, err := snapshot.Phase() + require.NoError(s.T(), err) + + return actualEpoch == expectedEpoch && actualPhase == expectedPhase + } + require.Eventuallyf(s.T(), condition, waitFor, tick, "did not reach expectedEpoch %d phase %s within %s", expectedEpoch, expectedPhase, waitFor) +} + +// AssertInEpochPhase checks if we are in the phase of the given epoch. +func (s *Suite) AssertInEpochPhase(ctx context.Context, expectedEpoch uint64, expectedPhase flow.EpochPhase) { snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) require.NoError(s.T(), err) + actualEpoch, err := snapshot.Epochs().Current().Counter() + require.NoError(s.T(), err) actualPhase, err := snapshot.Phase() require.NoError(s.T(), err) - require.Equal(s.T(), expectedPhase, actualPhase) + require.Equal(s.T(), expectedPhase, actualPhase, "not in correct phase") + require.Equal(s.T(), expectedEpoch, actualEpoch, "not in correct epoch") + + head, err := snapshot.Head() + require.NoError(s.T(), err) + s.TimedLogf("asserted in epoch %d, phase %s, finalized height/view: %d/%d", expectedEpoch, expectedPhase, head.Height, head.View) } -// assertEpochCounter requires actual epoch counter is equal to counter provided -func (s *Suite) assertEpochCounter(ctx context.Context, expectedCounter uint64) { +// AssertInEpoch requires actual epoch counter is equal to counter provided. +func (s *Suite) AssertInEpoch(ctx context.Context, expectedEpoch uint64) { snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) require.NoError(s.T(), err) - actualCounter, err := snapshot.Epochs().Current().Counter() + actualEpoch, err := snapshot.Epochs().Current().Counter() require.NoError(s.T(), err) - require.Equalf(s.T(), expectedCounter, actualCounter, "expected to be in epoch %d got %d", expectedCounter, actualCounter) + require.Equalf(s.T(), expectedEpoch, actualEpoch, "expected to be in epoch %d got %d", expectedEpoch, actualEpoch) } -// assertLatestFinalizedBlockHeightHigher will assert that the difference between snapshot height and latest finalized height -// is greater than numOfBlocks. -func (s *Suite) assertLatestFinalizedBlockHeightHigher(ctx context.Context, snapshot *inmem.Snapshot, numOfBlocks uint64) { - bootstrapHead, err := snapshot.Head() +// AssertNodeNotParticipantInEpoch asserts that the given node ID does not exist +// in the epoch's identity table. +func (s *Suite) AssertNodeNotParticipantInEpoch(epoch protocol.Epoch, nodeID flow.Identifier) { + identities, err := epoch.InitialIdentities() require.NoError(s.T(), err) + require.NotContains(s.T(), identities.NodeIDs(), nodeID) +} + +// AwaitSealedBlockHeightExceedsSnapshot polls until it observes that the latest +// sealed block height has exceeded the snapshot height by numOfBlocks +// the snapshot height and latest finalized height is greater than numOfBlocks. +func (s *Suite) AwaitSealedBlockHeightExceedsSnapshot(ctx context.Context, snapshot *inmem.Snapshot, threshold uint64, waitFor, tick time.Duration) { + header, err := snapshot.Head() + require.NoError(s.T(), err) + snapshotHeight := header.Height + + require.Eventually(s.T(), func() bool { + latestSealed := s.getLatestSealedHeader(ctx) + s.TimedLogf("waiting for sealed block height: %d+%d < %d", snapshotHeight, threshold, latestSealed.Height) + return snapshotHeight+threshold < latestSealed.Height + }, waitFor, tick) +} - header, err := s.client.GetLatestSealedBlockHeader(ctx) +// AwaitFinalizedView polls until it observes that the latest finalized block has a view +// greater than or equal to the input view. This is used to wait until when an epoch +// transition must have happened. +func (s *Suite) AwaitFinalizedView(ctx context.Context, view uint64, waitFor, tick time.Duration) { + require.Eventually(s.T(), func() bool { + sealed := s.getLatestFinalizedHeader(ctx) + return sealed.View >= view + }, waitFor, tick) +} + +// getLatestSealedHeader retrieves the latest sealed block, as reported in LatestSnapshot. +func (s *Suite) getLatestSealedHeader(ctx context.Context) *flow.Header { + snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) require.NoError(s.T(), err) + segment, err := snapshot.SealingSegment() + require.NoError(s.T(), err) + sealed := segment.Sealed() + return sealed.Header +} - // head should now be at-least numOfBlocks blocks higher from when we started - require.True(s.T(), header.Height-bootstrapHead.Height >= numOfBlocks, fmt.Sprintf("expected head.Height %d to be higher than head from the snapshot the node was bootstraped with bootstrapHead.Height %d.", header.Height, bootstrapHead.Height)) +// getLatestFinalizedHeader retrieves the latest finalized block, as reported in LatestSnapshot. +func (s *Suite) getLatestFinalizedHeader(ctx context.Context) *flow.Header { + snapshot, err := s.client.GetLatestProtocolSnapshot(ctx) + require.NoError(s.T(), err) + finalized, err := snapshot.Head() + require.NoError(s.T(), err) + return finalized } // submitSmokeTestTransaction will submit a create account transaction to smoke test network @@ -670,13 +564,14 @@ func (s *Suite) submitSmokeTestTransaction(ctx context.Context) { require.NoError(s.T(), err) } -// assertNetworkHealthyAfterANChange after an access node is removed or added to the network -// this func can be used to perform sanity. -// 1. Check that there is no problem connecting directly to the AN provided and retrieve a protocol snapshot -// 2. Check that the chain moved atleast 20 blocks from when the node was bootstrapped by comparing -// head of the rootSnapshot with the head of the snapshot we retrieved directly from the AN -// 3. Check that we can execute a script on the AN -func (s *Suite) assertNetworkHealthyAfterANChange(ctx context.Context, env templates.Environment, rootSnapshot *inmem.Snapshot, info *StakedNodeOperationInfo) { +// assertNetworkHealthyAfterANChange performs a basic network health check after replacing an access node. +// 1. Check that there is no problem connecting directly to the AN provided and retrieve a protocol snapshot +// 2. Check that the chain moved at least 20 blocks from when the node was bootstrapped by comparing +// head of the rootSnapshot with the head of the snapshot we retrieved directly from the AN +// 3. Check that we can execute a script on the AN +// +// TODO test sending and observing result of a transaction via the new AN (blocked by https://github.com/onflow/flow-go/issues/3642) +func (s *Suite) assertNetworkHealthyAfterANChange(ctx context.Context, env templates.Environment, snapshotInSecondEpoch *inmem.Snapshot, info *StakedNodeOperationInfo) { // get snapshot directly from new AN and compare head with head from the // snapshot that was used to bootstrap the node @@ -687,7 +582,7 @@ func (s *Suite) assertNetworkHealthyAfterANChange(ctx context.Context, env templ // overwrite client to point to the new AN (since we have stopped the initial AN at this point) s.client = client // assert atleast 20 blocks have been finalized since the node replacement - s.assertLatestFinalizedBlockHeightHigher(ctx, rootSnapshot, 20) + s.AwaitSealedBlockHeightExceedsSnapshot(ctx, snapshotInSecondEpoch, 10, 30*time.Second, time.Millisecond*100) // execute script directly on new AN to ensure it's functional proposedTable, err := client.ExecuteScriptBytes(ctx, templates.GenerateReturnProposedTableScript(env), []cadence.Value{}) @@ -695,33 +590,33 @@ func (s *Suite) assertNetworkHealthyAfterANChange(ctx context.Context, env templ require.Contains(s.T(), proposedTable.(cadence.Array).Values, cadence.String(info.NodeID.String()), "expected node ID to be present in proposed table returned by new AN.") } -// assertNetworkHealthyAfterVNChange after an verification node is removed or added to the network -// this func can be used to perform sanity. -// 1. Ensure sealing continues by comparing latest sealed block from the root snapshot to the current latest sealed block -func (s *Suite) assertNetworkHealthyAfterVNChange(ctx context.Context, _ templates.Environment, rootSnapshot *inmem.Snapshot, _ *StakedNodeOperationInfo) { - // assert at least 20 blocks have been finalized since the node replacement - s.assertLatestFinalizedBlockHeightHigher(ctx, rootSnapshot, 20) +// assertNetworkHealthyAfterVNChange performs a basic network health check after replacing a verification node. +// 1. Ensure sealing continues into the second epoch (post-replacement) by observing +// at least 10 blocks of sealing progress within the epoch +func (s *Suite) assertNetworkHealthyAfterVNChange(ctx context.Context, _ templates.Environment, snapshotInSecondEpoch *inmem.Snapshot, _ *StakedNodeOperationInfo) { + s.AwaitSealedBlockHeightExceedsSnapshot(ctx, snapshotInSecondEpoch, 10, 30*time.Second, time.Millisecond*100) } -// assertNetworkHealthyAfterLNChange after an collection node is removed or added to the network -// this func can be used to perform sanity. -// 1. Submit transaction to network that will target the newly staked LN by making sure the reference block ID -// is after the first epoch. +// assertNetworkHealthyAfterLNChange performs a basic network health check after replacing a collection node. +// 1. Submit transaction to network that will target the newly staked LN by making +// sure the reference block ID is after the first epoch. func (s *Suite) assertNetworkHealthyAfterLNChange(ctx context.Context, _ templates.Environment, _ *inmem.Snapshot, _ *StakedNodeOperationInfo) { - // At this point we have reached epoch 1 and our new LN node should be the only LN node in the network. - // To validate the LN joined the network successfully and is processing transactions we submit a - // create account transaction and assert there are no errors. + // At this point we have reached the second epoch and our new LN is the only LN in the network. + // To validate the LN joined the network successfully and is processing transactions we create + // an account, which submits a transaction and verifies it is sealed. s.submitSmokeTestTransaction(ctx) } -// assertNetworkHealthyAfterSNChange after replacing a consensus node in the test and waiting until -// the epoch transition we should observe blocks finalizing and we should be able to submit a transaction -// that will indicate overall network health -// 1. Submit transaction to network +// assertNetworkHealthyAfterSNChange performs a basic network health check after replacing a consensus node. +// The runTestEpochJoinAndLeave function running prior to this health check already asserts that we successfully: +// 1. enter the second epoch (DKG succeeds; epoch fallback is not triggered) +// 2. seal at least the first block within the second epoch (consensus progresses into second epoch). +// +// The test is configured so that one offline committee member is enough to prevent progress, +// therefore the newly joined consensus node must be participating in consensus. +// +// In addition, here, we submit a transaction and verify that it is sealed. func (s *Suite) assertNetworkHealthyAfterSNChange(ctx context.Context, _ templates.Environment, _ *inmem.Snapshot, _ *StakedNodeOperationInfo) { - // At this point we can assure that our SN node is participating in finalization and sealing because - // there are only 2 SN nodes in the network now we will submit a transaction to the - // network to ensure the network is overall healthy. s.submitSmokeTestTransaction(ctx) } @@ -761,44 +656,50 @@ func (s *Suite) runTestEpochJoinAndLeave(role flow.Role, checkNetworkHealth node // wait for epoch setup phase before we start our container and pause the old container s.TimedLogf("waiting for EpochSetup phase of first epoch to begin") - s.WaitForPhase(s.ctx, flow.EpochPhaseSetup) + s.AwaitEpochPhase(s.ctx, 0, flow.EpochPhaseSetup, 3*time.Minute, 500*time.Millisecond) s.TimedLogf("successfully reached EpochSetup phase of first epoch") - // get latest snapshot and start new container - snapshot, err := s.client.GetLatestProtocolSnapshot(s.ctx) + // get the latest snapshot and start new container with it + rootSnapshot, err := s.client.GetLatestProtocolSnapshot(s.ctx) require.NoError(s.T(), err) - testContainer.WriteRootSnapshot(snapshot) - testContainer.Container.Start(s.ctx) - header, err := snapshot.Head() + header, err := rootSnapshot.Head() + require.NoError(s.T(), err) + segment, err := rootSnapshot.SealingSegment() require.NoError(s.T(), err) - s.TimedLogf("retrieved header after entering EpochSetup phase: height=%d, view=%d", header.Height, header.View) - epoch1FinalView, err := snapshot.Epochs().Current().FinalView() + s.TimedLogf("retrieved header after entering EpochSetup phase: root_height=%d, root_view=%d, segment_heights=[%d-%d], segment_views=[%d-%d]", + header.Height, header.View, + segment.Sealed().Header.Height, segment.Highest().Header.Height, + segment.Sealed().Header.View, segment.Highest().Header.Height) + + testContainer.WriteRootSnapshot(rootSnapshot) + testContainer.Container.Start(s.ctx) + + epoch1FinalView, err := rootSnapshot.Epochs().Current().FinalView() require.NoError(s.T(), err) - // wait for 5 views after the start of the next epoch before we pause our container to replace - s.TimedLogf("waiting for sealed view %d before pausing container", epoch1FinalView+5) - s.BlockState.WaitForSealedView(s.T(), epoch1FinalView+5) - s.TimedLogf("observed sealed view %d -> pausing container", epoch1FinalView+5) + // wait for at least the first block of the next epoch to be sealed before we pause our container to replace + s.TimedLogf("waiting for epoch transition (finalized view %d) before pausing container", epoch1FinalView+1) + s.AwaitFinalizedView(s.ctx, epoch1FinalView+1, 4*time.Minute, 500*time.Millisecond) + s.TimedLogf("observed finalized view %d -> pausing container", epoch1FinalView+1) - // make sure container to replace removed from smart contract state - s.assertNodeNotApprovedOrProposed(s.ctx, env, containerToReplace.Config.NodeID) + // make sure container to replace is not a member of epoch 2 + s.AssertNodeNotParticipantInEpoch(rootSnapshot.Epochs().Next(), containerToReplace.Config.NodeID) // assert transition to second epoch happened as expected // if counter is still 0, epoch emergency fallback was triggered and we can fail early - s.assertEpochCounter(s.ctx, 1) + s.AssertInEpoch(s.ctx, 1) err = containerToReplace.Pause() require.NoError(s.T(), err) - // wait for 5 views after pausing our container to replace before we assert healthy network - s.TimedLogf("waiting for sealed view %d before asserting network health", epoch1FinalView+10) - s.BlockState.WaitForSealedView(s.T(), epoch1FinalView+10) - s.TimedLogf("observed sealed view %d -> asserting network health", epoch1FinalView+10) + // retrieve a snapshot after observing that we have entered the second epoch + secondEpochSnapshot, err := s.client.GetLatestProtocolSnapshot(s.ctx) + require.NoError(s.T(), err) // make sure the network is healthy after adding new node - checkNetworkHealth(s.ctx, env, snapshot, info) + checkNetworkHealth(s.ctx, env, secondEpochSnapshot, info) } // DynamicEpochTransitionSuite is the suite used for epoch transitions tests @@ -808,11 +709,14 @@ type DynamicEpochTransitionSuite struct { } func (s *DynamicEpochTransitionSuite) SetupTest() { - // use a longer staking auction length to accommodate staking operations for - // joining/leaving nodes - s.StakingAuctionLen = 200 + // use a longer staking auction length to accommodate staking operations for joining/leaving nodes + // NOTE: this value is set fairly aggressively to ensure shorter test times. + // If flakiness due to failure to complete staking operations in time is observed, + // try increasing (by 10-20 views). + s.StakingAuctionLen = 50 s.DKGPhaseLen = 50 - s.EpochLen = 380 + s.EpochLen = 250 + s.EpochCommitSafetyThreshold = 20 // run the generic setup, which starts up the network s.Suite.SetupTest() diff --git a/integration/tests/execution/suite.go b/integration/tests/execution/suite.go index 68e0b5727ae..8c27d3e0de2 100644 --- a/integration/tests/execution/suite.go +++ b/integration/tests/execution/suite.go @@ -124,7 +124,6 @@ func (s *Suite) SetupTest() { for _, nodeID := range s.nodeIDs { nodeConfig := testnet.NewNodeConfig(flow.RoleConsensus, testnet.WithID(nodeID), testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag(blockRateFlag), ) s.nodeConfigs = append(s.nodeConfigs, nodeConfig) diff --git a/integration/tests/lib/block_state.go b/integration/tests/lib/block_state.go index 6b8bf804c66..a0cb14ee5cc 100644 --- a/integration/tests/lib/block_state.go +++ b/integration/tests/lib/block_state.go @@ -273,27 +273,6 @@ func (bs *BlockState) HighestSealed() (*flow.Block, bool) { return bs.highestSealed, true } -func (bs *BlockState) WaitForSealedView(t *testing.T, view uint64) *flow.Block { - timeout := 3 * blockStateTimeout - require.Eventually(t, - func() bool { - bs.RLock() - defer bs.RUnlock() - - if bs.highestSealed != nil { - t.Logf("%v waiting for sealed view (%d/%d)", time.Now().UTC(), bs.highestSealed.Header.View, view) - } - return bs.highestSealed != nil && bs.highestSealed.Header.View >= view - }, - timeout, - 100*time.Millisecond, - fmt.Sprintf("did not receive sealed block for view (%v) within %v seconds", view, timeout)) - - bs.RLock() - defer bs.RUnlock() - return bs.highestSealed -} - func (bs *BlockState) FinalizedHeight(currentHeight uint64) (*flow.Block, bool) { bs.RLock() defer bs.RUnlock() diff --git a/integration/tests/lib/util.go b/integration/tests/lib/util.go index 6cfb92d1569..af5a3e4f37d 100644 --- a/integration/tests/lib/util.go +++ b/integration/tests/lib/util.go @@ -4,8 +4,12 @@ import ( "context" "fmt" "math/rand" + "testing" + "time" "github.com/onflow/cadence" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" @@ -224,3 +228,49 @@ func WithChainID(chainID flow.ChainID) func(tx *sdk.Transaction) { } } } + +// LogStatus logs current information about the test network state. +func LogStatus(t *testing.T, ctx context.Context, log zerolog.Logger, client *testnet.Client) { + snapshot, err := client.GetLatestProtocolSnapshot(ctx) + if err != nil { + log.Err(err).Msg("failed to get sealed snapshot") + return + } + finalized, err := client.GetLatestFinalizedBlockHeader(ctx) + if err != nil { + log.Err(err).Msg("failed to get finalized header") + return + } + + sealed, err := snapshot.Head() + require.NoError(t, err) + phase, err := snapshot.Phase() + require.NoError(t, err) + epoch := snapshot.Epochs().Current() + counter, err := epoch.Counter() + require.NoError(t, err) + + log.Info().Uint64("final_height", finalized.Height). + Uint64("sealed_height", sealed.Height). + Uint64("sealed_view", sealed.View). + Str("cur_epoch_phase", phase.String()). + Uint64("cur_epoch_counter", counter). + Msg("test run status") +} + +// LogStatusPeriodically periodically logs information about the test network state. +// It can be run as a goroutine at the beginning of a test run to provide period +func LogStatusPeriodically(t *testing.T, parent context.Context, log zerolog.Logger, client *testnet.Client, period time.Duration) { + log = log.With().Str("util", "status_logger").Logger() + for { + select { + case <-parent.Done(): + return + case <-time.After(period): + } + + ctx, cancel := context.WithTimeout(parent, 30*time.Second) + LogStatus(t, ctx, log, client) + cancel() + } +} diff --git a/integration/tests/mvp/mvp_test.go b/integration/tests/mvp/mvp_test.go index ba521cc25c8..166c87688ad 100644 --- a/integration/tests/mvp/mvp_test.go +++ b/integration/tests/mvp/mvp_test.go @@ -115,13 +115,11 @@ func TestMVP_Bootstrap(t *testing.T) { func buildMVPNetConfig() testnet.NetworkConfig { collectionConfigs := []func(*testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), testnet.WithLogLevel(zerolog.FatalLevel), } consensusConfigs := []func(config *testnet.NodeConfig){ - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--block-rate-delay=100ms"), testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", 1)), testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", 1)), diff --git a/integration/tests/verification/suite.go b/integration/tests/verification/suite.go index 1616a8d02e9..4ce6092513f 100644 --- a/integration/tests/verification/suite.go +++ b/integration/tests/verification/suite.go @@ -77,7 +77,6 @@ func (s *Suite) SetupSuite() { nodeConfig := testnet.NewNodeConfig(flow.RoleConsensus, testnet.WithID(nodeID), testnet.WithLogLevel(zerolog.FatalLevel), - testnet.WithAdditionalFlag("--hotstuff-timeout=12s"), testnet.WithAdditionalFlag("--required-verification-seal-approvals=1"), testnet.WithAdditionalFlag("--required-construction-seal-approvals=1"), testnet.WithAdditionalFlag(blockRateFlag), diff --git a/integration/tests/verification/verify_system_chunk_test.go b/integration/tests/verification/verify_system_chunk_test.go index 1bdb574d445..f972adaa23d 100644 --- a/integration/tests/verification/verify_system_chunk_test.go +++ b/integration/tests/verification/verify_system_chunk_test.go @@ -46,8 +46,13 @@ func (suite *VerifySystemChunkSuite) TestSystemChunkIDsShouldBeDifferent() { require.Equal(suite.T(), stateA, stateB) // computes ids of system chunk for result A and B - systemChunkAId := receiptA.ExecutionResult.Chunks[0].ID() - systemChunkBId := receiptB.ExecutionResult.Chunks[0].ID() + systemChunkA := receiptA.ExecutionResult.Chunks[0] + systemChunkAId := systemChunkA.ID() + suite.T().Logf("system chunk for blockA: %v\n", *systemChunkA) + + systemChunkB := receiptB.ExecutionResult.Chunks[0] + systemChunkBId := systemChunkB.ID() + suite.T().Logf("system chunk for blockB: %v\n", *systemChunkB) // requires that system chunk Id of execution results be different require.NotEqual(suite.T(), systemChunkAId, systemChunkBId) diff --git a/integration/utils/templates/create-and-setup-node.cdc b/integration/utils/templates/create-and-setup-node.cdc new file mode 100644 index 00000000000..0a631e9276d --- /dev/null +++ b/integration/utils/templates/create-and-setup-node.cdc @@ -0,0 +1,60 @@ +import Crypto +import FungibleToken from 0xFUNGIBLETOKENADDRESS +import FlowToken from 0xFLOWTOKENADDRESS +import FlowIDTableStaking from 0xIDENTITYTABLEADDRESS +import FlowStakingCollection from 0xSTAKINGCOLLECTIONADDRESS + +transaction( + stakingAcctKey: Crypto.KeyListEntry, + stake: UFix64, + id: String, + role: UInt8, + networkingAddress: String, + networkingKey: String, + stakingKey: String, + machineAcctKey: Crypto.KeyListEntry?) { + + prepare(service: AuthAccount) { + // 1 - create the staking account for the new node. + // + let stakingAccount = AuthAccount(payer: service) + stakingAccount.keys.add(publicKey: stakingAcctKey.publicKey, hashAlgorithm: stakingAcctKey.hashAlgorithm, weight: stakingAcctKey.weight) + + // 2 - fund the new staking account + // + let stakeDst = stakingAccount.getCapability(/public/flowTokenReceiver).borrow<&{FungibleToken.Receiver}>() + ?? panic("Could not borrow receiver reference to the recipient's Vault") + // withdraw stake from service account + let stakeSrc = service.borrow<&FlowToken.Vault>(from: /storage/flowTokenVault) + ?? panic("Could not borrow reference to the owner's Vault!") + stakeDst.deposit(from: <-stakeSrc.withdraw(amount: stake)) + + // 3 - set up the staking collection + // + let flowToken = stakingAccount.link<&FlowToken.Vault>(/private/flowTokenVault, target: /storage/flowTokenVault)! + // Create a new Staking Collection and put it in storage + let stakingCollection <-FlowStakingCollection.createStakingCollection(unlockedVault: flowToken, tokenHolder: nil) + let stakingCollectionRef = &stakingCollection as &FlowStakingCollection.StakingCollection + stakingAccount.save(<-stakingCollection, to: FlowStakingCollection.StakingCollectionStoragePath) + + // Create a public link to the staking collection + stakingAccount.link <&FlowStakingCollection.StakingCollection{FlowStakingCollection.StakingCollectionPublic}> ( + FlowStakingCollection.StakingCollectionPublicPath, + target: FlowStakingCollection.StakingCollectionStoragePath + ) + + // 4 - register the node + // + if let machineAccount = stakingCollectionRef.registerNode( + id: id, + role: role, + networkingAddress: networkingAddress, + networkingKey: networkingKey, + stakingKey: stakingKey, + amount: stake, + payer: service, + ) { + machineAccount.keys.add(publicKey: machineAcctKey!.publicKey, hashAlgorithm: machineAcctKey!.hashAlgorithm, weight: machineAcctKey!.weight) + } + } +} diff --git a/integration/utils/templates/remove-node.cdc b/integration/utils/templates/remove-node.cdc new file mode 100644 index 00000000000..88679d076ec --- /dev/null +++ b/integration/utils/templates/remove-node.cdc @@ -0,0 +1,26 @@ +import FlowIDTableStaking from 0xIDENTITYTABLEADDRESS + +// This transaction removes an existing node from the identity table +// by unstaking it and removing it from the approved list +transaction(id: String) { + + // Local variable for a reference to the ID Table Admin object + let adminRef: &FlowIDTableStaking.Admin + + prepare(acct: AuthAccount) { + // borrow a reference to the admin object + self.adminRef = acct.borrow<&FlowIDTableStaking.Admin>(from: FlowIDTableStaking.StakingAdminStoragePath) + ?? panic("Could not borrow reference to staking admin") + } + + execute { + self.adminRef.removeAndRefundNodeRecord(id) + let nodeIDs = FlowIDTableStaking.getApprovedList() + nodeIDs[id] = nil + + // set the approved list to the new allow-list + self.adminRef.setApprovedList(nodeIDs) + } +} + + diff --git a/integration/utils/transactions.go b/integration/utils/transactions.go index 866b0197670..26e1eb2012a 100644 --- a/integration/utils/transactions.go +++ b/integration/utils/transactions.go @@ -1,18 +1,22 @@ package utils import ( - "fmt" + _ "embed" "github.com/onflow/cadence" "github.com/onflow/flow-core-contracts/lib/go/templates" - fttemplates "github.com/onflow/flow-ft/lib/go/templates" sdk "github.com/onflow/flow-go-sdk" - "github.com/onflow/flow-go-sdk/crypto" sdktemplates "github.com/onflow/flow-go-sdk/templates" "github.com/onflow/flow-go/model/flow" ) +//go:embed templates/create-and-setup-node.cdc +var createAndSetupNodeTxScript string + +//go:embed templates/remove-node.cdc +var removeNodeTxScript string + func LocalnetEnv() templates.Environment { return templates.Environment{ IDTableAddress: "f8d6e0586b0a20c7", @@ -25,103 +29,109 @@ func LocalnetEnv() templates.Environment { } } -// MakeCreateStakingCollectionTx submits transaction stakingCollection/setup_staking_collection.cdc -// which will setup the staking collection object for the account -func MakeCreateStakingCollectionTx( - env templates.Environment, - stakingAccount *sdk.Account, - stakingAccountKeyID int, - stakingSigner crypto.Signer, - payerAddress sdk.Address, - latestBlockID sdk.Identifier, -) (*sdk.Transaction, error) { - accountKey := stakingAccount.Keys[stakingAccountKeyID] - tx := sdk.NewTransaction(). - SetScript(templates.GenerateCollectionSetup(env)). - SetGasLimit(9999). - SetReferenceBlockID(latestBlockID). - SetProposalKey(stakingAccount.Address, stakingAccountKeyID, accountKey.SequenceNumber). - SetPayer(payerAddress). - AddAuthorizer(stakingAccount.Address) - - //signing the payload as used AddAuthorizer - err := tx.SignPayload(stakingAccount.Address, stakingAccountKeyID, stakingSigner) - if err != nil { - return nil, fmt.Errorf("could not sign payload: %w", err) - } - - return tx, nil -} - -func MakeStakingCollectionRegisterNodeTx( +// MakeCreateAndSetupNodeTx creates a transaction which creates and configures a new staking account. +// It creates the account, funds it, and registers the node using the staking collection. +func MakeCreateAndSetupNodeTx( env templates.Environment, - stakingAccount *sdk.Account, - stakingAccountKeyID int, - stakingSigner crypto.Signer, - payerAddress sdk.Address, + service *sdk.Account, latestBlockID sdk.Identifier, + // transaction arguments + stakingAcctKey *sdk.AccountKey, + stake string, nodeID flow.Identifier, role flow.Role, networkingAddress string, networkingKey string, stakingKey string, - amount string, machineKey *sdk.AccountKey, -) (*sdk.Transaction, error) { - accountKey := stakingAccount.Keys[stakingAccountKeyID] +) ( + *sdk.Transaction, + error, +) { + + script := []byte(templates.ReplaceAddresses(createAndSetupNodeTxScript, env)) tx := sdk.NewTransaction(). - SetScript(templates.GenerateCollectionRegisterNode(env)). + SetScript(script). SetGasLimit(9999). SetReferenceBlockID(latestBlockID). - SetProposalKey(stakingAccount.Address, stakingAccountKeyID, accountKey.SequenceNumber). - SetPayer(payerAddress). - AddAuthorizer(stakingAccount.Address) + SetProposalKey(service.Address, 0, service.Keys[0].SequenceNumber). + AddAuthorizer(service.Address). + SetPayer(service.Address) - id, _ := cadence.NewString(nodeID.String()) - err := tx.AddArgument(id) + // 0 - staking account key + cdcStakingAcctKey, err := sdktemplates.AccountKeyToCadenceCryptoKey(stakingAcctKey) + if err != nil { + return nil, err + } + err = tx.AddArgument(cdcStakingAcctKey) if err != nil { return nil, err } + // 1 - stake + cdcStake, err := cadence.NewUFix64(stake) + if err != nil { + return nil, err + } + err = tx.AddArgument(cdcStake) + if err != nil { + return nil, err + } + + // 2 - node ID + id, err := cadence.NewString(nodeID.String()) + if err != nil { + return nil, err + } + err = tx.AddArgument(id) + if err != nil { + return nil, err + } + + // 3 - role r := cadence.NewUInt8(uint8(role)) err = tx.AddArgument(r) if err != nil { return nil, err } - networkingAddressCDC, _ := cadence.NewString(networkingAddress) + // 4 - networking address + networkingAddressCDC, err := cadence.NewString(networkingAddress) + if err != nil { + return nil, err + } err = tx.AddArgument(networkingAddressCDC) if err != nil { return nil, err } - networkingKeyCDC, _ := cadence.NewString(networkingKey) + // 5 - networking key + networkingKeyCDC, err := cadence.NewString(networkingKey) + if err != nil { + return nil, err + } err = tx.AddArgument(networkingKeyCDC) if err != nil { return nil, err } - stakingKeyCDC, _ := cadence.NewString(stakingKey) - err = tx.AddArgument(stakingKeyCDC) + // 6 - staking key + stakingKeyCDC, err := cadence.NewString(stakingKey) if err != nil { return nil, err } - - amountCDC, _ := cadence.NewUFix64(amount) - err = tx.AddArgument(amountCDC) + err = tx.AddArgument(stakingKeyCDC) if err != nil { return nil, err } if machineKey != nil { // for collection/consensus nodes, register the machine account key - publicKey, err := sdktemplates.AccountKeyToCadenceCryptoKey(machineKey) + cdcMachineAcctKey, err := sdktemplates.AccountKeyToCadenceCryptoKey(machineKey) if err != nil { return nil, err } - publicKeysCDC := cadence.NewArray([]cadence.Value{publicKey}) - - err = tx.AddArgument(cadence.NewOptional(publicKeysCDC)) + err = tx.AddArgument(cadence.NewOptional(cdcMachineAcctKey)) if err != nil { return nil, err } @@ -133,52 +143,11 @@ func MakeStakingCollectionRegisterNodeTx( } } - err = tx.SignPayload(stakingAccount.Address, stakingAccountKeyID, stakingSigner) - if err != nil { - return nil, fmt.Errorf("could not sign payload: %w", err) - } - - return tx, nil -} - -func MakeStakingCollectionCloseStakeTx( - env templates.Environment, - stakingAccount *sdk.Account, - stakingAccountKeyID int, - stakingSigner crypto.Signer, - payerAddress sdk.Address, - latestBlockID sdk.Identifier, - nodeID flow.Identifier, -) (*sdk.Transaction, error) { - accountKey := stakingAccount.Keys[stakingAccountKeyID] - tx := sdk.NewTransaction(). - SetScript(templates.GenerateCollectionCloseStake(env)). - SetGasLimit(9999). - SetReferenceBlockID(latestBlockID). - SetProposalKey(stakingAccount.Address, stakingAccountKeyID, accountKey.SequenceNumber). - SetPayer(payerAddress). - AddAuthorizer(stakingAccount.Address) - - id, _ := cadence.NewString(nodeID.String()) - err := tx.AddArgument(id) - if err != nil { - return nil, err - } - - err = tx.AddArgument(cadence.NewOptional(nil)) - if err != nil { - return nil, err - } - - err = tx.SignPayload(stakingAccount.Address, stakingAccountKeyID, stakingSigner) - if err != nil { - return nil, fmt.Errorf("could not sign payload: %w", err) - } - return tx, nil } -// MakeAdminRemoveNodeTx makes the admin remove node transaction. This is equivalent to the node un-staking and will result in removal at the next epoch boundary +// MakeAdminRemoveNodeTx makes an admin transaction to remove the node. This transaction both +// manually unstakes and removes the node, and removes it from the allow-list. func MakeAdminRemoveNodeTx( env templates.Environment, adminAccount *sdk.Account, @@ -188,7 +157,7 @@ func MakeAdminRemoveNodeTx( ) (*sdk.Transaction, error) { accountKey := adminAccount.Keys[adminAccountKeyID] tx := sdk.NewTransaction(). - SetScript(templates.GenerateRemoveNodeScript(env)). + SetScript([]byte(templates.ReplaceAddresses(removeNodeTxScript, env))). SetGasLimit(9999). SetReferenceBlockID(latestBlockID). SetProposalKey(adminAccount.Address, adminAccountKeyID, accountKey.SequenceNumber). @@ -203,35 +172,3 @@ func MakeAdminRemoveNodeTx( return tx, nil } - -func MakeTransferTokenTx(env templates.Environment, receiver sdk.Address, sender *sdk.Account, senderKeyID int, tokenAmount string, latestBlockID sdk.Identifier) ( - *sdk.Transaction, error) { - - senderKey := sender.Keys[senderKeyID] - fungible := sdk.HexToAddress(env.FungibleTokenAddress) - flowToken := sdk.HexToAddress(env.FlowTokenAddress) - script := fttemplates.GenerateTransferVaultScript(fungible, flowToken, "FlowToken") - - tx := sdk.NewTransaction(). - SetScript([]byte(script)). - SetGasLimit(100). - SetReferenceBlockID(latestBlockID). - SetProposalKey(sender.Address, senderKeyID, senderKey.SequenceNumber). - SetPayer(sender.Address). - AddAuthorizer(sender.Address) - - amount, err := cadence.NewUFix64(tokenAmount) - if err != nil { - return nil, fmt.Errorf("could not add arguments to transaction :%w", err) - } - err = tx.AddArgument(amount) - if err != nil { - return nil, fmt.Errorf("could not add argument to transaction :%w", err) - } - - err = tx.AddArgument(cadence.NewAddress(receiver)) - if err != nil { - return nil, fmt.Errorf("could not add argument to transaction :%w", err) - } - return tx, nil -} diff --git a/ledger/complete/mtrie/trie/trie.go b/ledger/complete/mtrie/trie/trie.go index 70c718b2155..b2ec3106f5d 100644 --- a/ledger/complete/mtrie/trie/trie.go +++ b/ledger/complete/mtrie/trie/trie.go @@ -94,9 +94,9 @@ func (mt *MTrie) RootNode() *node.Node { // Concurrency safe (as Tries are immutable structures by convention) func (mt *MTrie) String() string { if mt.IsEmpty() { - return fmt.Sprintf("Empty Trie with default root hash: %x\n", mt.RootHash()) + return fmt.Sprintf("Empty Trie with default root hash: %v\n", mt.RootHash()) } - trieStr := fmt.Sprintf("Trie root hash: %x\n", mt.RootHash()) + trieStr := fmt.Sprintf("Trie root hash: %v\n", mt.RootHash()) return trieStr + mt.root.FmtStr("", "") } @@ -364,12 +364,18 @@ type updateResult struct { lowestHeightTouched int } -// update traverses the subtree, updates the stored registers, and returns: -// - new or original node (n) +// update traverses the subtree recursively and create new nodes with +// the updated payloads on the given paths +// +// it returns: +// - new updated node or original node if nothing was updated // - allocated register count delta in subtrie (allocatedRegCountDelta) // - allocated register size delta in subtrie (allocatedRegSizeDelta) // - lowest height reached during recursive update in subtrie (lowestHeightTouched) // +// update also compact a subtree into a single compact leaf node in the case where +// there is only 1 payload stored in the subtree. +// // allocatedRegCountDelta and allocatedRegSizeDelta are used to compute updated // trie's allocated register count and size. lowestHeightTouched is used to // compute max depth touched during update. @@ -379,70 +385,82 @@ type updateResult struct { // (excluding the bit at index headHeight) // - paths are NOT duplicated func update( - nodeHeight int, parentNode *node.Node, - paths []ledger.Path, payloads []ledger.Payload, compactLeaf *node.Node, - prune bool, + nodeHeight int, // the height of the node during traversing the subtree + currentNode *node.Node, // the current node on the travesing path, if it's nil it means the trie has no node on this path + paths []ledger.Path, // the paths to update the payloads + payloads []ledger.Payload, // the payloads to be updated at the given paths + compactLeaf *node.Node, // a compact leaf node from its ancester, it could be nil + prune bool, // prune is a flag for whether pruning nodes with empty payload. not pruning is useful for generating proof, expecially non-inclusion proof ) (n *node.Node, allocatedRegCountDelta int64, allocatedRegSizeDelta int64, lowestHeightTouched int) { - // No new paths to write + // No new path to update if len(paths) == 0 { - // check is a compactLeaf from a higher height is still left. if compactLeaf != nil { - // create a new node for the compact leaf path and payload. The old node shouldn't - // be recycled as it is still used by the tree copy before the update. + // if a compactLeaf from a higher height is still left, + // then expand the compact leaf node to the current height by creating a new compact leaf + // node with the same path and payload. + // The old node shouldn't be recycled as it is still used by the tree copy before the update. n = node.NewLeaf(*compactLeaf.Path(), compactLeaf.Payload(), nodeHeight) return n, 0, 0, nodeHeight } - return parentNode, 0, 0, nodeHeight + // if no path to update and there is no compact leaf node on this path, we return + // the current node regardless it exists or not. + return currentNode, 0, 0, nodeHeight } - if len(paths) == 1 && parentNode == nil && compactLeaf == nil { + if len(paths) == 1 && currentNode == nil && compactLeaf == nil { + // if there is only 1 path to update, and the existing tree has no node on this path, also + // no compact leaf node from its ancester, it means we are storing a payload on a new path, n = node.NewLeaf(paths[0], payloads[0].DeepCopy(), nodeHeight) if payloads[0].IsEmpty() { - // Unallocated register doesn't affect allocatedRegCountDelta and allocatedRegSizeDelta. + // if we are storing an empty node, then no register is allocated + // allocatedRegCountDelta and allocatedRegSizeDelta should both be 0 return n, 0, 0, nodeHeight } + // if we are storing a non-empty node, we are allocating a new register return n, 1, int64(payloads[0].Size()), nodeHeight } - if parentNode != nil && parentNode.IsLeaf() { // if we're here then compactLeaf == nil - // check if the parent node path is among the updated paths + if currentNode != nil && currentNode.IsLeaf() { // if we're here then compactLeaf == nil + // check if the current node path is among the updated paths found := false - parentPath := *parentNode.Path() + currentPath := *currentNode.Path() for i, p := range paths { - if p == parentPath { + if p == currentPath { // the case where the recursion stops: only one path to update if len(paths) == 1 { - if !parentNode.Payload().ValueEquals(&payloads[i]) { + // check if the only path to update has the same payload. + // if payload is the same, we could skip the update to avoid creating duplicated node + if !currentNode.Payload().ValueEquals(&payloads[i]) { n = node.NewLeaf(paths[i], payloads[i].DeepCopy(), nodeHeight) allocatedRegCountDelta, allocatedRegSizeDelta = - computeAllocatedRegDeltas(parentNode.Payload(), &payloads[i]) + computeAllocatedRegDeltas(currentNode.Payload(), &payloads[i]) return n, allocatedRegCountDelta, allocatedRegSizeDelta, nodeHeight } // avoid creating a new node when the same payload is written - return parentNode, 0, 0, nodeHeight + return currentNode, 0, 0, nodeHeight } // the case where the recursion carries on: len(paths)>1 found = true allocatedRegCountDelta, allocatedRegSizeDelta = - computeAllocatedRegDeltasFromHigherHeight(parentNode.Payload()) + computeAllocatedRegDeltasFromHigherHeight(currentNode.Payload()) break } } if !found { - // if the parent node carries a path not included in the input path, then the parent node + // if the current node carries a path not included in the input path, then the current node // represents a compact leaf that needs to be carried down the recursion. - compactLeaf = parentNode + compactLeaf = currentNode } } - // in the remaining code: the registers to update are strictly larger than 1: - // - either len(paths)>1 + // in the remaining code: + // - either len(paths) > 1 // - or len(paths) == 1 and compactLeaf!= nil - // - or len(paths) == 1 and parentNode != nil && !parentNode.IsLeaf() + // - or len(paths) == 1 and currentNode != nil && !currentNode.IsLeaf() // Split paths and payloads to recurse: // lpaths contains all paths that have `0` at the partitionIndex @@ -464,25 +482,25 @@ func update( } } - // set the parent node children - var lchildParent, rchildParent *node.Node - if parentNode != nil { - lchildParent = parentNode.LeftChild() - rchildParent = parentNode.RightChild() + // set the node children + var oldLeftChild, oldRightChild *node.Node + if currentNode != nil { + oldLeftChild = currentNode.LeftChild() + oldRightChild = currentNode.RightChild() } // recurse over each branch - var lChild, rChild *node.Node + var newLeftChild, newRightChild *node.Node var lRegCountDelta, rRegCountDelta int64 var lRegSizeDelta, rRegSizeDelta int64 var lLowestHeightTouched, rLowestHeightTouched int parallelRecursionThreshold := 16 if len(lpaths) < parallelRecursionThreshold || len(rpaths) < parallelRecursionThreshold { // runtime optimization: if there are _no_ updates for either left or right sub-tree, proceed single-threaded - lChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = update(nodeHeight-1, lchildParent, lpaths, lpayloads, lcompactLeaf, prune) - rChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, rchildParent, rpaths, rpayloads, rcompactLeaf, prune) + newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) + newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) } else { - // runtime optimization: process the left child is a separate thread + // runtime optimization: process the left child in a separate thread // Since we're receiving 4 values from goroutine, use a // struct and channel to reduce allocs/op. @@ -491,15 +509,15 @@ func update( // channel is faster and uses fewer allocs/op in this case. results := make(chan updateResult, 1) go func(retChan chan<- updateResult) { - child, regCountDelta, regSizeDelta, lowestHeightTouched := update(nodeHeight-1, lchildParent, lpaths, lpayloads, lcompactLeaf, prune) + child, regCountDelta, regSizeDelta, lowestHeightTouched := update(nodeHeight-1, oldLeftChild, lpaths, lpayloads, lcompactLeaf, prune) retChan <- updateResult{child, regCountDelta, regSizeDelta, lowestHeightTouched} }(results) - rChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, rchildParent, rpaths, rpayloads, rcompactLeaf, prune) + newRightChild, rRegCountDelta, rRegSizeDelta, rLowestHeightTouched = update(nodeHeight-1, oldRightChild, rpaths, rpayloads, rcompactLeaf, prune) // Wait for results from goroutine. ret := <-results - lChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.allocatedRegSizeDelta, ret.lowestHeightTouched + newLeftChild, lRegCountDelta, lRegSizeDelta, lLowestHeightTouched = ret.child, ret.allocatedRegCountDelta, ret.allocatedRegSizeDelta, ret.lowestHeightTouched } allocatedRegCountDelta += lRegCountDelta + rRegCountDelta @@ -510,18 +528,20 @@ func update( // payload is re-written at a register. CAUTION: we only check that the children are // unchanged. This is only sufficient for interim nodes (for leaf nodes, the children // might be unchanged, i.e. both nil, but the payload could have changed). - if !parentNode.IsLeaf() && lChild == lchildParent && rChild == rchildParent { - return parentNode, 0, 0, lowestHeightTouched + // In case the current node was a leaf, we _cannot reuse_ it, because we potentially + // updated registers in the sub-trie + if !currentNode.IsLeaf() && newLeftChild == oldLeftChild && newRightChild == oldRightChild { + return currentNode, 0, 0, lowestHeightTouched } - // In case the parent node was a leaf, we _cannot reuse_ it, because we potentially - // updated registers in the sub-trie + // if prune is on, then will check and create a compact leaf node if one child is nil, and the + // other child is a leaf node if prune { - n = node.NewInterimCompactifiedNode(nodeHeight, lChild, rChild) + n = node.NewInterimCompactifiedNode(nodeHeight, newLeftChild, newRightChild) return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched } - n = node.NewInterimNode(nodeHeight, lChild, rChild) + n = node.NewInterimNode(nodeHeight, newLeftChild, newRightChild) return n, allocatedRegCountDelta, allocatedRegSizeDelta, lowestHeightTouched } diff --git a/ledger/partial/ledger_test.go b/ledger/partial/ledger_test.go index d14c1e7d4c1..77da5d28912 100644 --- a/ledger/partial/ledger_test.go +++ b/ledger/partial/ledger_test.go @@ -119,11 +119,14 @@ func TestProofsForEmptyRegisters(t *testing.T) { // create empty update emptyState := l.InitialState() - view := delta.NewView(executionState.LedgerGetRegister(l, flow.StateCommitment(emptyState))) + view := delta.NewDeltaView( + executionState.NewLedgerStorageSnapshot( + l, + flow.StateCommitment(emptyState))) registerID := flow.NewRegisterID("b", "nk") - v, err := view.Get(registerID.Owner, registerID.Key) + v, err := view.Get(registerID) require.NoError(t, err) require.Empty(t, v) @@ -133,7 +136,7 @@ func TestProofsForEmptyRegisters(t *testing.T) { updated, err := ledger.NewUpdate(emptyState, keys, values) require.NoError(t, err) - allRegisters := view.Interactions().AllRegisters() + allRegisters := view.Interactions().AllRegisterIDs() allKeys := make([]ledger.Key, len(allRegisters)) for i, id := range allRegisters { allKeys[i] = executionState.RegisterIDToKey(id) diff --git a/model/chainsync/status.go b/model/chainsync/status.go index 9153d22074e..153109e7068 100644 --- a/model/chainsync/status.go +++ b/model/chainsync/status.go @@ -1,5 +1,3 @@ -// (c) 2019 Dapper Labs - ALL RIGHTS RESERVED - package chainsync import ( @@ -50,6 +48,7 @@ func (s *Status) StatusString() string { func NewQueuedStatus(height uint64) *Status { return &Status{ - Queued: time.Now(), + BlockHeight: height, + Queued: time.Now(), } } diff --git a/model/convert/fixtures/fixture.go b/model/convert/fixtures/fixture.go index cb59647824a..75b6eb9a982 100644 --- a/model/convert/fixtures/fixture.go +++ b/model/convert/fixtures/fixture.go @@ -19,7 +19,7 @@ func EpochSetupFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochSetu } event := unittest.EventFixture(events.EpochSetup.EventType(), 1, 1, unittest.IdentifierFixture(), 0) - event.Payload = []byte(epochSetupFixtureJSON) + event.Payload = []byte(EpochSetupFixtureJSON) // randomSource is [0,0,...,1,2,3,4] randomSource := make([]uint8, flow.EpochSetupRandomSourceLength) @@ -118,7 +118,7 @@ func EpochCommitFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochCom } event := unittest.EventFixture(events.EpochCommit.EventType(), 1, 1, unittest.IdentifierFixture(), 0) - event.Payload = []byte(epochCommitFixtureJSON) + event.Payload = []byte(EpochCommitFixtureJSON) expected := &flow.EpochCommit{ Counter: 1, @@ -147,7 +147,7 @@ func EpochCommitFixtureByChainID(chain flow.ChainID) (flow.Event, *flow.EpochCom return event, expected } -var epochSetupFixtureJSON = ` +var EpochSetupFixtureJSON = ` { "type": "Event", "value": { @@ -1079,7 +1079,7 @@ var epochSetupFixtureJSON = ` } ` -var epochCommitFixtureJSON = ` +var EpochCommitFixtureJSON = ` { "type": "Event", "value": { diff --git a/model/flow/address.go b/model/flow/address.go index 022f741cbb2..a0b054f28fb 100644 --- a/model/flow/address.go +++ b/model/flow/address.go @@ -9,38 +9,20 @@ import ( "strings" ) +// AddressLength is the size of an account address in bytes. +// (n) is the size of an account address in bits. +const AddressLength = (linearCodeN + 7) >> 3 + // Address represents the 8 byte address of an account. type Address [AddressLength]byte -type AddressGenerator interface { - NextAddress() (Address, error) - CurrentAddress() Address - Bytes() []byte - AddressCount() uint64 // returns the total number of addresses that have been generated so far -} - -type MonotonicAddressGenerator struct { - index uint64 -} +// EmptyAddress is the default value of a variable of type Address +var EmptyAddress = BytesToAddress(nil) -// linearCodeAddressGenerator represents the internal index of the linear code address generation mechanism -type linearCodeAddressGenerator struct { - chainCodeWord uint64 - index uint64 +func ConvertAddress(b [AddressLength]byte) Address { + return Address(b) } -const ( - // AddressLength is the size of an account address in bytes. - // (n) is the size of an account address in bits. - AddressLength = (linearCodeN + 7) >> 3 - // addressIndexLength is the size of an account address state in bytes. - // (k) is the size of an account address in bits. - addressIndexLength = (linearCodeK + 7) >> 3 -) - -// EmptyAddress is the default value of a variable of type Address -var EmptyAddress = Address{} - // HexToAddress converts a hex string to an Address. func HexToAddress(h string) Address { trimmed := strings.TrimPrefix(h, "0x") @@ -105,6 +87,12 @@ func (a *Address) UnmarshalJSON(data []byte) error { return nil } +// uint64 converts an address into a uint64 +func (a Address) uint64() uint64 { + v := binary.BigEndian.Uint64(a[:]) + return v +} + // modified from binary.bigEndian.uint64 func uint48(b []byte) uint64 { _ = b[5] // bounds check hint to compiler; @@ -123,12 +111,33 @@ func putUint48(b []byte, v uint64) { b[5] = byte(v) } +// addressIndexLength is the size of an account address state in bytes. +// (k) is the size of an account address in bits. +const addressIndexLength = (linearCodeK + 7) >> 3 + func indexToBytes(index uint64) []byte { indexBytes := make([]byte, addressIndexLength) putUint48(indexBytes, index) return indexBytes } +type AddressGenerator interface { + NextAddress() (Address, error) + CurrentAddress() Address + Bytes() []byte + AddressCount() uint64 // returns the total number of addresses that have been generated so far +} + +type MonotonicAddressGenerator struct { + index uint64 +} + +// linearCodeAddressGenerator represents the internal index of the linear code address generation mechanism +type linearCodeAddressGenerator struct { + chainCodeWord uint64 + index uint64 +} + // Bytes converts an address index into a slice of bytes func (gen *MonotonicAddressGenerator) Bytes() []byte { return indexToBytes(gen.index) @@ -218,12 +227,6 @@ func uint64ToAddress(v uint64) Address { return Address(b) } -// uint64 converts an address into a uint64 -func (a *Address) uint64() uint64 { - v := binary.BigEndian.Uint64(a[:]) - return v -} - const ( // [n,k,d]-Linear code parameters // The linear code used in the account addressing is a [64,45,7] diff --git a/model/flow/address_test.go b/model/flow/address_test.go index edfb10eda24..b71a0d567ed 100644 --- a/model/flow/address_test.go +++ b/model/flow/address_test.go @@ -7,6 +7,9 @@ import ( "testing" "time" + "github.com/onflow/cadence" + "github.com/onflow/cadence/runtime/common" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -15,6 +18,34 @@ type addressWrapper struct { Address Address } +func TestConvertAddress(t *testing.T) { + expected := BytesToAddress([]byte{1, 2, 3, 4, 5, 6, 7, 8}) + cadenceAddress := cadence.BytesToAddress([]byte{1, 2, 3, 4, 5, 6, 7, 8}) + runtimeAddress := common.MustBytesToAddress([]byte{1, 2, 3, 4, 5, 6, 7, 8}) + + assert.NotEqual(t, cadenceAddress, runtimeAddress) + + assert.Equal(t, expected, ConvertAddress(cadenceAddress)) + assert.Equal(t, expected, ConvertAddress(runtimeAddress)) +} + +func TestBytesToAddress(t *testing.T) { + type testCase struct { + expected string + value string + } + + for _, test := range []testCase{ + {string([]byte{0, 0, 0, 0, 0, 0, 0, 0}), ""}, + {string([]byte{0, 0, 0, 0}) + "1234", "1234"}, + {"12345678", "12345678"}, + {"12345678", "trim12345678"}, + } { + address := BytesToAddress([]byte(test.value)) + assert.Equal(t, test.expected, string(address[:])) + } +} + func TestHexToAddress(t *testing.T) { type testCase struct { diff --git a/model/flow/block.go b/model/flow/block.go index 711c09bc421..229a6059dcb 100644 --- a/model/flow/block.go +++ b/model/flow/block.go @@ -5,10 +5,7 @@ package flow func Genesis(chainID ChainID) *Block { // create the raw content for the genesis block - payload := Payload{ - Guarantees: nil, - Seals: nil, - } + payload := Payload{} // create the header header := Header{ diff --git a/model/flow/constants.go b/model/flow/constants.go index dbf7b133dbe..a109f912497 100644 --- a/model/flow/constants.go +++ b/model/flow/constants.go @@ -10,7 +10,7 @@ var GenesisTime = time.Date(2018, time.December, 19, 22, 32, 30, 42, time.UTC) // DefaultProtocolVersion is the default protocol version, indicating none was // explicitly set during bootstrapping. -const DefaultProtocolVersion = 0 +const DefaultProtocolVersion uint = 0 // DefaultTransactionExpiry is the default expiry for transactions, measured in blocks. // The default value is equivalent to 10 minutes for a 1-second block time. diff --git a/model/flow/epoch.go b/model/flow/epoch.go index 60f97a8ea18..3f27586f2a2 100644 --- a/model/flow/epoch.go +++ b/model/flow/epoch.go @@ -411,6 +411,10 @@ type EpochStatus struct { PreviousEpoch EventIDs // EpochSetup and EpochCommit events for the previous epoch CurrentEpoch EventIDs // EpochSetup and EpochCommit events for the current epoch NextEpoch EventIDs // EpochSetup and EpochCommit events for the next epoch + // InvalidServiceEventIncorporated encodes whether an invalid service event is + // incorporated in this fork. When this happens, epoch fallback is triggered + // AFTER the fork is finalized. + InvalidServiceEventIncorporated bool } // Copy returns a copy of the epoch status. diff --git a/model/flow/execution_result.go b/model/flow/execution_result.go index c16d10b6752..9661ff3d3c2 100644 --- a/model/flow/execution_result.go +++ b/model/flow/execution_result.go @@ -17,6 +17,22 @@ type ExecutionResult struct { ExecutionDataID Identifier } +func NewExecutionResult( + previousResultID Identifier, + blockID Identifier, + chunks ChunkList, + serviceEvents ServiceEventList, + executionDataID Identifier, +) *ExecutionResult { + return &ExecutionResult{ + PreviousResultID: previousResultID, + BlockID: blockID, + Chunks: chunks, + ServiceEvents: serviceEvents, + ExecutionDataID: executionDataID, + } +} + // ID returns the hash of the execution result body func (er ExecutionResult) ID() Identifier { return MakeID(er) diff --git a/model/flow/header.go b/model/flow/header.go index 5b505749e47..665d0a3854b 100644 --- a/model/flow/header.go +++ b/model/flow/header.go @@ -15,30 +15,36 @@ import ( // the combined payload of the entire block. It is what consensus nodes agree // on after validating the contents against the payload hash. type Header struct { - ChainID ChainID // ChainID is a chain-specific value to prevent replay attacks. - - ParentID Identifier // ParentID is the ID of this block's parent. - - Height uint64 // Height is the height of the parent + 1 - - PayloadHash Identifier // PayloadHash is a hash of the payload of this block. - - Timestamp time.Time // Timestamp is the time at which this block was proposed. + // ChainID is a chain-specific value to prevent replay attacks. + ChainID ChainID + // ParentID is the ID of this block's parent. + ParentID Identifier + // Height is the height of the parent + 1 + Height uint64 + // PayloadHash is a hash of the payload of this block. + PayloadHash Identifier + // Timestamp is the time at which this block was proposed. // The proposer can choose any time, so this should not be trusted as accurate. - - View uint64 // View number at which this block was proposed. - - ParentVoterIndices []byte // a bitvector that represents all the voters for the parent block. - - ParentVoterSigData []byte // aggregated signature over the parent block. Not a single cryptographic + Timestamp time.Time + // View number at which this block was proposed. + View uint64 + // ParentView number at which parent block was proposed. + ParentView uint64 + // ParentVoterIndices is a bitvector that represents all the voters for the parent block. + ParentVoterIndices []byte + // ParentVoterSigData is an aggregated signature over the parent block. Not a single cryptographic // signature since the data represents cryptographic signatures serialized in some way (concatenation or other) // A quorum certificate can be extracted from the header. // This field is the SigData field of the extracted quorum certificate. - - ProposerID Identifier // proposer identifier for the block - - ProposerSigData []byte // signature of the proposer over the new block. Not a single cryptographic + ParentVoterSigData []byte + // ProposerID is a proposer identifier for the block + ProposerID Identifier + // ProposerSigData is a signature of the proposer over the new block. Not a single cryptographic // signature since the data represents cryptographic signatures serialized in some way (concatenation or other) + ProposerSigData []byte + // LastViewTC is a timeout certificate for previous view, it can be nil + // it has to be present if previous round ended with timeout. + LastViewTC *TimeoutCertificate } // Body returns the immutable part of the block header. @@ -50,9 +56,11 @@ func (h Header) Body() interface{} { PayloadHash Identifier Timestamp uint64 View uint64 + ParentView uint64 ParentVoterIndices []byte ParentVoterSigData []byte ProposerID Identifier + LastViewTCID Identifier }{ ChainID: h.ChainID, ParentID: h.ParentID, @@ -60,9 +68,11 @@ func (h Header) Body() interface{} { PayloadHash: h.PayloadHash, Timestamp: uint64(h.Timestamp.UnixNano()), View: h.View, + ParentView: h.ParentView, ParentVoterIndices: h.ParentVoterIndices, ParentVoterSigData: h.ParentVoterSigData, ProposerID: h.ProposerID, + LastViewTCID: h.LastViewTC.ID(), } } diff --git a/model/flow/header_test.go b/model/flow/header_test.go index 4273890c87c..cbc6b2fc272 100644 --- a/model/flow/header_test.go +++ b/model/flow/header_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/vmihailenco/msgpack/v4" + "github.com/onflow/flow-go/consensus/hotstuff/helper" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/encoding/rlp" "github.com/onflow/flow-go/model/flow" @@ -31,6 +32,7 @@ func TestHeaderEncodingJSON(t *testing.T) { func TestHeaderFingerprint(t *testing.T) { header := unittest.BlockHeaderFixture() + header.LastViewTC = helper.MakeTC() headerID := header.ID() data := header.Fingerprint() var decoded struct { @@ -40,9 +42,11 @@ func TestHeaderFingerprint(t *testing.T) { PayloadHash flow.Identifier Timestamp uint64 View uint64 + ParentView uint64 ParentVoterIndices []byte ParentVoterSigData crypto.Signature ProposerID flow.Identifier + LastViewTC interface{} } rlp.NewMarshaler().MustUnmarshal(data, &decoded) decHeader := &flow.Header{ @@ -52,10 +56,12 @@ func TestHeaderFingerprint(t *testing.T) { PayloadHash: decoded.PayloadHash, Timestamp: time.Unix(0, int64(decoded.Timestamp)).UTC(), View: decoded.View, + ParentView: decoded.ParentView, ParentVoterIndices: decoded.ParentVoterIndices, ParentVoterSigData: decoded.ParentVoterSigData, ProposerID: decoded.ProposerID, - ProposerSigData: header.ProposerSigData, // since this field is not encoded/decoded, just set it to the original + ProposerSigData: header.ProposerSigData, // since this field is not encoded/decoded, just set it to the original value to pass test + LastViewTC: header.LastViewTC, } decodedID := decHeader.ID() assert.Equal(t, headerID, decodedID) diff --git a/model/flow/ledger.go b/model/flow/ledger.go index 0b5f0f23c5e..e6517b5d2ff 100644 --- a/model/flow/ledger.go +++ b/model/flow/ledger.go @@ -1,40 +1,144 @@ package flow import ( + "encoding/binary" "encoding/hex" "encoding/json" "fmt" + "strings" "github.com/onflow/flow-go/ledger/common/hash" "github.com/onflow/flow-go/model/fingerprint" ) +const ( + // Service level keys (owner is empty): + UUIDKey = "uuid" + AddressStateKey = "account_address_state" + + // Account level keys + AccountKeyPrefix = "a." + AccountStatusKey = AccountKeyPrefix + "s" + CodeKeyPrefix = "code." + ContractNamesKey = "contract_names" + PublicKeyKeyPrefix = "public_key_" +) + +func addressToOwner(address Address) string { + return string(address.Bytes()) +} + type RegisterID struct { Owner string Key string } -// String returns formatted string representation of the RegisterID. -// TODO(rbtz): remove after the merge of https://github.com/onflow/flow-emulator/pull/235 -// Deprecated: used only by emultor. -func (r *RegisterID) String() string { - return fmt.Sprintf("%x/%x", r.Owner, r.Key) +var AddressStateRegisterID = RegisterID{ + Owner: "", + Key: AddressStateKey, } -// Bytes returns a bytes representation of the RegisterID. -// -// The encoding uses the injective fingerprint module. -func (r *RegisterID) Bytes() []byte { - return fingerprint.Fingerprint(r) +var UUIDRegisterID = RegisterID{ + Owner: "", + Key: UUIDKey, +} + +func AccountStatusRegisterID(address Address) RegisterID { + return RegisterID{ + Owner: addressToOwner(address), + Key: AccountStatusKey, + } +} + +func PublicKeyRegisterID(address Address, index uint64) RegisterID { + return RegisterID{ + Owner: addressToOwner(address), + Key: fmt.Sprintf("public_key_%d", index), + } +} + +func ContractNamesRegisterID(address Address) RegisterID { + return RegisterID{ + Owner: addressToOwner(address), + Key: ContractNamesKey, + } +} + +func ContractRegisterID(address Address, contractName string) RegisterID { + return RegisterID{ + Owner: addressToOwner(address), + Key: CodeKeyPrefix + contractName, + } +} + +func CadenceRegisterID(owner []byte, key []byte) RegisterID { + return RegisterID{ + Owner: string(BytesToAddress(owner).Bytes()), + Key: string(key), + } } func NewRegisterID(owner, key string) RegisterID { return RegisterID{ - Owner: owner, + Owner: addressToOwner(BytesToAddress([]byte(owner))), Key: key, } } +// IsInternalState returns true if the register id is controlled by flow-go and +// return false otherwise (key controlled by the cadence env). +func (id RegisterID) IsInternalState() bool { + // check if is a service level key (owner is empty) + // cases: + // - "", "uuid" + // - "", "account_address_state" + if len(id.Owner) == 0 && (id.Key == UUIDKey || id.Key == AddressStateKey) { + return true + } + + // check account level keys + // cases: + // - address, "contract_names" + // - address, "code.%s" (contract name) + // - address, "public_key_%d" (index) + // - address, "a.s" (account status) + return strings.HasPrefix(id.Key, PublicKeyKeyPrefix) || + id.Key == ContractNamesKey || + strings.HasPrefix(id.Key, CodeKeyPrefix) || + id.Key == AccountStatusKey +} + +// IsSlabIndex returns true if the key is a slab index for an account's ordered fields +// map. +// +// In general, each account's regular fields are stored in ordered map known +// only to cadence. Cadence encodes this map into bytes and split the bytes +// into slab chunks before storing the slabs into the ledger. +func (id RegisterID) IsSlabIndex() bool { + return len(id.Key) == 9 && id.Key[0] == '$' +} + +// TODO(patrick): pretty print flow internal register ids. +// String returns formatted string representation of the RegisterID. +func (id RegisterID) String() string { + formattedKey := "" + if id.IsSlabIndex() { + i := uint64(binary.BigEndian.Uint64([]byte(id.Key[1:]))) + formattedKey = fmt.Sprintf("$%d", i) + } else { + formattedKey = fmt.Sprintf("#%x", []byte(id.Key)) + } + + return fmt.Sprintf("%x/%s", id.Owner, formattedKey) +} + +// Bytes returns a bytes representation of the RegisterID. +// +// The encoding uses the injective fingerprint module. +func (r *RegisterID) Bytes() []byte { + return fingerprint.Fingerprint(r) +} + // RegisterValue (value part of Register) type RegisterValue = []byte diff --git a/model/flow/ledger_test.go b/model/flow/ledger_test.go index c5a615399ee..13d18e2977d 100644 --- a/model/flow/ledger_test.go +++ b/model/flow/ledger_test.go @@ -3,8 +3,11 @@ package flow import ( "encoding/hex" "fmt" - "testing" + "unicode/utf8" + + "github.com/onflow/atree" + "github.com/stretchr/testify/require" ) // this benchmark can run with this command: @@ -15,10 +18,7 @@ var length int func BenchmarkString(b *testing.B) { - var r = RegisterID{ - Owner: "theowner", - Key: "123412341234", - } + r := NewRegisterID("theowner", "123412341234") ownerLen := len(r.Owner) @@ -38,12 +38,54 @@ func BenchmarkString(b *testing.B) { func BenchmarkOriginalString(b *testing.B) { - var r = RegisterID{ - Owner: "theowner", - Key: "123412341234", - } + r := NewRegisterID("theowner", "123412341234") ret := fmt.Sprintf("%x/%x", r.Owner, r.Key) length = len(ret) } + +func TestRegisterID_IsInternalState(t *testing.T) { + requireTrue := func(owner string, key string) { + id := NewRegisterID(owner, key) + require.True(t, id.IsInternalState()) + } + + requireFalse := func(owner string, key string) { + id := NewRegisterID(owner, key) + require.False(t, id.IsInternalState()) + } + require.True(t, UUIDRegisterID.IsInternalState()) + requireFalse("", UUIDKey) + require.True(t, AddressStateRegisterID.IsInternalState()) + requireFalse("", AddressStateKey) + requireFalse("", "other") + requireFalse("Address", UUIDKey) + requireFalse("Address", AddressStateKey) + requireTrue("Address", "public_key_12") + requireTrue("Address", ContractNamesKey) + requireTrue("Address", "code.MYCODE") + requireTrue("Address", AccountStatusKey) + requireFalse("Address", "anything else") +} + +func TestRegisterID_String(t *testing.T) { + // slab with 189 should result in \\xbd + slabIndex := atree.StorageIndex([8]byte{0, 0, 0, 0, 0, 0, 0, 189}) + + id := NewRegisterID( + string([]byte{1, 2, 3, 10}), + string(atree.SlabIndexToLedgerKey(slabIndex))) + require.False(t, utf8.ValidString(id.Key)) + printable := id.String() + require.True(t, utf8.ValidString(printable)) + require.Equal(t, "000000000102030a/$189", printable) + + // non slab invalid utf-8 + id = NewRegisterID("b\xc5y", "a\xc5z") + require.False(t, utf8.ValidString(id.Owner)) + require.False(t, utf8.ValidString(id.Key)) + printable = id.String() + require.True(t, utf8.ValidString(printable)) + require.Equal(t, "000000000062c579/#61c57a", printable) +} diff --git a/model/flow/payload.go b/model/flow/payload.go index 1e9b86e4706..a6af04000a3 100644 --- a/model/flow/payload.go +++ b/model/flow/payload.go @@ -6,11 +6,16 @@ import ( // Payload is the actual content of each block. type Payload struct { - // Guarantees are ordered in execution order. + // Guarantees are ordered in execution order. May be empty, in which case + // only the system chunk is executed for this block. Guarantees []*CollectionGuarantee - Seals []*Seal - Receipts ExecutionReceiptMetaList - Results ExecutionResultList + // Seals holds block seals for ancestor blocks. + // The oldest seal must connect to the latest seal in the fork extended by this block. + // Seals must be internally connected, containing no seals with duplicate block IDs or heights. + // Seals may be empty. It presents a set, i.e. there is no protocol-defined ordering. + Seals []*Seal + Receipts ExecutionReceiptMetaList + Results ExecutionResultList } // EmptyPayload returns an empty block payload. diff --git a/model/flow/quorum_certificate.go b/model/flow/quorum_certificate.go index a74924693b3..3fac30f4cf1 100644 --- a/model/flow/quorum_certificate.go +++ b/model/flow/quorum_certificate.go @@ -22,6 +22,14 @@ type QuorumCertificate struct { SigData []byte } +// ID returns the QuorumCertificate's identifier +func (qc *QuorumCertificate) ID() Identifier { + if qc == nil { + return ZeroID + } + return MakeID(qc) +} + // QuorumCertificateWithSignerIDs is a QuorumCertificate, where the signing nodes are // identified via their `flow.Identifier`s instead of indices. Working with IDs as opposed to // indices is less efficient, but simpler, because we don't require a canonical node order. diff --git a/model/flow/sealing_segment.go b/model/flow/sealing_segment.go index 5f4cab6c5de..e0a04cb9eec 100644 --- a/model/flow/sealing_segment.go +++ b/model/flow/sealing_segment.go @@ -2,69 +2,39 @@ package flow import ( "fmt" + + "golang.org/x/exp/slices" ) -// SealingSegment is the chain segment such that the last block (greatest height) -// is this snapshot's reference block and the first (least height) is the most -// recently sealed block as of this snapshot (ie. the block referenced by LatestSeal). -// -// In other words, the most recently incorporated seal as of the highest block -// references the lowest block. The highest block does not need to contain this seal. -// -// Example 1 - E seals A: -// -// A <- B <- C <- D <- E(SA) -// -// The above sealing segment's last block (E) has a seal for block A, which is -// the first block of the sealing segment. -// -// Example 2 - E contains no seals, but latest seal prior to E seals A: -// -// A <- B <- C <- D(SA) <- E -// -// Example 3 - E contains multiple seals -// -// B <- C <- D <- E(SA, SB) -// -// MINIMALITY REQUIREMENT: -// Note that block B is the highest sealed block as of E. Therefore, the -// sealing segment's lowest block must be B. Essentially, this is a minimality -// requirement for the history: it shouldn't be longer than necessary. So -// extending the chain segment above to A <- B <- C <- D <- E(SA, SB) would -// _not_ yield a valid SealingSegment. +// SealingSegment is a continuous segment of recently finalized blocks that contains enough history +// for a new node to execute its business logic normally. It is part of the data need to initialize +// a new node to join the network. +// DETAILED SPECIFICATION: ./sealing_segment.md // -// ROOT SEALING SEGMENTS: -// Root sealing segments are sealing segments which contain the root block: -// * the root block is a self-sealing block with an empty payload -// * the root block must be the first block (least height) in the segment -// * no blocks in the segment may contain any seals (by the minimality requirement) -// * it is possible (but not necessary) for root sealing segments to contain only the root block -// -// Example 1 - one self-sealing root block -// -// ROOT -// -// The above sealing segment is the form of sealing segments within root snapshots, -// for example those snapshots used to bootstrap a new network, or spork. +// ├═══════════┤ ├───────────────────────┤ +// ExtraBlocks ^ Blocks ^ +// B head // -// Example 2 - one self-sealing root block followed by any number of seal-less blocks -// -// ROOT <- A <- B -// -// All non-root sealing segments contain more than one block. -// Sealing segments are in ascending height order. -// -// In addition to storing the blocks within the sealing segment, as defined above, -// the SealingSegment structure also stores any resources which are referenced -// by blocks in the segment, but not included in the payloads of blocks within -// the segment. In particular: -// * results referenced by receipts within segment payloads -// * results referenced by seals within segment payloads -// * seals which represent the latest state commitment as of a segment block +// Lets denote the highest block in the sealing segment as `head`. Per convention, `head` must be a +// finalized block. Consider the chain of blocks leading up to `head` (included). The highest block +// in chain leading up to `head` that is sealed, we denote as B. type SealingSegment struct { - // Blocks contain the chain segment blocks in ascending height order. + // Blocks contain the chain `B <- ... <- Head` in ascending height order. + // Formally, Blocks contains exactly (not more!) the history to satisfy condition + // (see sealing_segment.md for details): + // (i) The highest sealed block as of `head` needs to be included in the sealing segment. + // This is relevant if `head` does not contain any seals. Blocks []*Block + // ExtraBlocks [optional] holds ancestors of `Blocks` in ascending height order. + // Formally, ExtraBlocks contains at least the additional history to satisfy conditions + // (see sealing_segment.md for details): + // (ii) All blocks that are sealed by `head`. This is relevant if `head` contains _multiple_ seals. + // (iii) The sealing segment holds the history of all non-expired collection guarantees, i.e. + // limitHeight := max(head.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + // (Potentially longer history is permitted) + ExtraBlocks []*Block + // ExecutionResults contain any results which are referenced by receipts // or seals in the sealing segment, but not included in any segment block // payloads. @@ -89,14 +59,22 @@ type SealingSegment struct { FirstSeal *Seal } +// Highest is the highest block in the sealing segment and the reference block from snapshot that was +// used to produce this sealing segment. func (segment *SealingSegment) Highest() *Block { return segment.Blocks[len(segment.Blocks)-1] } -func (segment *SealingSegment) Lowest() *Block { +// Sealed returns the most recently sealed block based on head of sealing segment(highest block). +func (segment *SealingSegment) Sealed() *Block { return segment.Blocks[0] } +// AllBlocks returns all blocks within the sealing segment, including extra blocks, in ascending height order. +func (segment *SealingSegment) AllBlocks() []*Block { + return append(segment.ExtraBlocks, segment.Blocks...) +} + // FinalizedSeal returns the seal that seals the lowest block. // Per specification, this seal must be included in a SealingSegment. // The SealingSegment must be validated. @@ -112,9 +90,9 @@ func (segment *SealingSegment) FinalizedSeal() (*Seal, error) { } // sanity check - if seal.BlockID != segment.Lowest().ID() { + if seal.BlockID != segment.Sealed().ID() { return nil, fmt.Errorf("finalized seal should seal the lowest block %v, but actually is to seal %v", - segment.Lowest().ID(), seal.BlockID) + segment.Sealed().ID(), seal.BlockID) } return seal, nil } @@ -168,6 +146,15 @@ func (segment *SealingSegment) Validate() error { return fmt.Errorf("invalid segment: %w", err) } } + // extra blocks should be added in reverse order, starting from the highest one since they are sorted + // in ascending order. + for i := len(segment.ExtraBlocks) - 1; i >= 0; i-- { + block := segment.ExtraBlocks[i] + err := builder.AddExtraBlock(block) + if err != nil { + return fmt.Errorf("invalid segment: %w", err) + } + } _, err := builder.SealingSegment() if err != nil { return fmt.Errorf("invalid segment: %w", err) @@ -204,11 +191,19 @@ type SealingSegmentBuilder struct { results []*ExecutionResult latestSeals map[Identifier]Identifier firstSeal *Seal + // extraBlocks included in sealing segment, must connect to the lowest block of segment + // stored in descending order for simpler population logic + extraBlocks []*Block } // AddBlock appends a block to the sealing segment under construction. // No errors are expected during normal operation. func (builder *SealingSegmentBuilder) AddBlock(block *Block) error { + // sanity check: all blocks have to be added before adding extra blocks + if len(builder.extraBlocks) > 0 { + return fmt.Errorf("cannot add sealing segment block after extra block is added") + } + // sanity check: block should be 1 height higher than current highest if !builder.isValidHeight(block) { return fmt.Errorf("invalid block height (%d): %w", block.Header.Height, ErrSegmentInvalidBlockHeight) @@ -274,6 +269,27 @@ func (builder *SealingSegmentBuilder) AddBlock(block *Block) error { return nil } +// AddExtraBlock appends an extra block to sealing segment under construction. +// Extra blocks needs to be added in descending order and the first block must connect to the lowest block +// of sealing segment, this way they form a continuous chain. +// No errors are expected during normal operation. +func (builder *SealingSegmentBuilder) AddExtraBlock(block *Block) error { + if len(builder.extraBlocks) == 0 { + if len(builder.blocks) == 0 { + return fmt.Errorf("cannot add extra blocks before adding lowest sealing segment block") + } + // first extra block has to match the lowest block of sealing segment + if (block.Header.Height + 1) != builder.lowest().Header.Height { + return fmt.Errorf("invalid extra block height (%d), doesn't connect to sealing segment: %w", block.Header.Height, ErrSegmentInvalidBlockHeight) + } + } else if (block.Header.Height + 1) != builder.extraBlocks[len(builder.extraBlocks)-1].Header.Height { + return fmt.Errorf("invalid extra block height (%d), doesn't connect to last extra block: %w", block.Header.Height, ErrSegmentInvalidBlockHeight) + } + + builder.extraBlocks = append(builder.extraBlocks, block) + return nil +} + // AddExecutionResult adds result to executionResults func (builder *SealingSegmentBuilder) addExecutionResult(result *ExecutionResult) { builder.results = append(builder.results, result) @@ -290,8 +306,15 @@ func (builder *SealingSegmentBuilder) SealingSegment() (*SealingSegment, error) return nil, fmt.Errorf("failed to validate sealing segment: %w", err) } + // SealingSegment must store extra blocks in ascending order, builder stores them in descending. + // Apply a sort to reverse the slice and use correct ordering. + slices.SortFunc(builder.extraBlocks, func(lhs, rhs *Block) bool { + return lhs.Header.Height < rhs.Header.Height + }) + return &SealingSegment{ Blocks: builder.blocks, + ExtraBlocks: builder.extraBlocks, ExecutionResults: builder.results, LatestSeals: builder.latestSeals, FirstSeal: builder.firstSeal, @@ -319,6 +342,9 @@ func (builder *SealingSegmentBuilder) validateRootSegment() error { if len(builder.blocks) == 0 { return fmt.Errorf("root segment must have at least 1 block") } + if len(builder.extraBlocks) > 0 { + return fmt.Errorf("root segment cannot have extra blocks") + } if builder.lowest().Header.View != 0 { return fmt.Errorf("root block has unexpected view (%d != 0)", builder.lowest().Header.View) } @@ -354,6 +380,12 @@ func (builder *SealingSegmentBuilder) validateSegment() error { return fmt.Errorf("expect at least 2 blocks in a sealing segment or 1 block in the case of root segments, but got an empty sealing segment: %w", ErrSegmentBlocksWrongLen) } + if len(builder.extraBlocks) > 0 { + if builder.extraBlocks[0].Header.Height+1 != builder.lowest().Header.Height { + return fmt.Errorf("extra blocks don't connect to lowest block in segment") + } + } + // if root sealing segment, use different validation if isRootSegment(builder.latestSeals) { err := builder.validateRootSegment() @@ -394,6 +426,7 @@ func NewSealingSegmentBuilder(resultLookup GetResultFunc, sealLookup GetSealByBl includedResults: make(map[Identifier]struct{}), latestSeals: make(map[Identifier]Identifier), blocks: make([]*Block, 0), + extraBlocks: make([]*Block, 0), results: make(ExecutionResultList, 0), } } diff --git a/model/flow/sealing_segment.md b/model/flow/sealing_segment.md new file mode 100644 index 00000000000..1cc6544ec37 --- /dev/null +++ b/model/flow/sealing_segment.md @@ -0,0 +1,151 @@ +# Sealing Segment + +The `SealingSegment` is a section of the finalized chain. It is part of the data need to +initialize a new node to join the network. Informally, the `SealingSegment` is continuous section +of recently finalized blocks that is long enough for the new node to execute its business logic. + +## History length covered by the Sealing Segment + +The `SealingSegment` is created from a `protocol.Snapshot` via the method `SealingSegment`. +Lets denote the block that the `protocol.Snapshot` refers to as `head`. Per convention, +`head` must be a finalized block. + +### Part 1: from `head` back to the latest sealed block + +The SealingSegment is a chain segment such that the last block (greatest height) +is this snapshot's reference block (i.e. `head`) and the first (least height) is the most +recently sealed block as of this snapshot. +In other words, the most recently incorporated seal as of the highest block +references the lowest block. The highest block does not need to contain this seal. +* Example 1: block `E` seals `A` + ``` + A <- B <- C <- D <- E(SA) + ``` + Here, `SA` denotes the seal for block `A`. + In the sealing segment's last block (`E`) has a seal for block `A`, which is the first block of the sealing segment. + +* Example 2: `E` contains no seals, but latest seal prior to `E` seals `A` + ``` + A <- B <- C <- D(SA) <- E + ``` +* Example 3: `E` contains multiple seals + ``` + B <- C <- D <- E(SA, SB) + ``` + +Per convention, the blocks from Part 1, go into the slice `SealingSegment.Blocks`: +```golang +type SealingSegment struct { + // Blocks contain the chain [block sealed by `head`] <- ... <- [`head`] in ascending height order. + Blocks []*Block + + ⋮ +} +``` + +**Minimality Requirement for `SealingSegment.Blocks`**: +In example 3, note that block `B` is the highest sealed block as of `E`. Therefore, the +lowest block in `SealingSegment.Blocks` must be `B`. Essentially, this is a minimality +requirement for the history: it shouldn't be longer than necessary. So +extending the chain segment above to `A <- B <- C <- D <- E(SA, SB)` would +be an invalid value for field `SealingSegment.Blocks`. + +### Part 2: `ExtraBlocks` + +In addition, the `SealingSegment` contains the field `ExtraBlocks`: + +```golang +// ExtraBlocks [optional] holds ancestors of `Blocks` in ascending height order. These blocks +// are connecting to `Blocks[0]` (the lowest block of sealing segment). Formally, let `l` +// be the length of `ExtraBlocks`, then ExtraBlocks[l-1] is the _parent_ of `Blocks[0]`. +// These extra blocks are included in order to ensure that a newly bootstrapped node +// knows about all entities which might be referenced by blocks which extend from +// the sealing segment. +// ExtraBlocks are stored separately from Blocks, because not all node roles need +// the same amount of history. Blocks represents the minimal possible required history; +// ExtraBlocks represents additional role-required history. +ExtraBlocks []*Block +``` + +**In case `head` contains multiple seals, we need _all_ the sealed blocks**, for the following reason: +* All nodes locally maintain a copy of the protocol state. A service event may change the state of the protocol state. +* For Byzantine resilience, we don't want protocol-state changes to take effect immediately. Therefore, we process + system events only after receiving a QC for the block. + + Now let us consider the situation where a newly initialized node comes online and processes the first child of `head`. + Lets reuse the example from above, where our head was block `E` and we are now processing the child `X` + ``` + A <- B <- C <- D <- E(SA, SB) <- X + ├═══════════┤ ├───────────────────────┤ new + ExtraBlocks Blocks block + ``` + `X` carries the QC for `E`, hence the protocol-state changes in `E` take effect for `X`. Therefore, when processing `X`, + we go through the seals in `E` and look through the sealed execution results for service events. +* As the service events are order-sensitive, we need to process the seals in the correct order, which is by increasing height + of the sealed block. The seals don't contain the block's height information, hence we need to resolve the block. + +**Extended history to check for duplicated collection guarantees in blocks** is required by nodes that _validate_ block +payloads (e.g. consensus nodes). Also Access Nodes require these blocks. Collections expire after `flow.DefaultTransactionExpiry` blocks. +Hence, we desire a history of `flow.DefaultTransactionExpiry` blocks. However, there is the edge case of a recent spork (or genesis), +where the history is simply less that `flow.DefaultTransactionExpiry`. + +### Formal definition + +The descriptions from the previous section can be formalized as follows + +* (i) The highest sealed block as of `head` needs to be included in the sealing segment. + This is relevant if `head` does not contain any seals. +* (ii) All blocks that are sealed by `head`. This is relevant if `head` contains _multiple_ seals. +* (iii) The sealing segment should contain the history back to (including): + ``` + limitHeight := max(head.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + ``` +Note that all three conditions have to be satisfied by a sealing segment. Therefore, it must contain the longest history +required by any of the three conditions. The 'Spork Root Block' is the cutoff. + +Per convention, we include the blocks for (i) in the `SealingSegment.Blocks`, while the +additional blocks for (ii) and optionally (iii) are contained in as `SealingSegment.ExtraBlocks`. + + +Condition (i) and (ii) are necessary for the sealing segment for _any node_. In contrast, (iii) is +necessary to bootstrap nodes that _validate_ block payloads (e.g. consensus nodes), to verify that +collection guarantees are not duplicated (collections expire after `flow.DefaultTransactionExpiry` blocks). + +## Special case: Root Sealing Segment + +The spork is initialized with a single 'spork root block'. A root sealing segment is a sealing segment containing root block: +* the root block is a self-sealing block with an empty payload +* the root block must be the first block (least height) in the segment +* no blocks in the segment may contain any seals (by the minimality requirement) +* it is possible (but not necessary) for root sealing segments to contain _only_ the root block + +Examples: +* Example 1: one self-sealing root block + ``` + ROOT + ``` + The above sealing segment is the form of sealing segments within root snapshots, + for example those snapshots used to bootstrap a new network, or spork. +* Example 2: one self-sealing root block followed by any number of seal-less blocks + ``` + ROOT <- A <- B + ``` + All non-root sealing segments contain more than one block. + Sealing segments are in ascending height order. + +In addition to storing the blocks within the sealing segment, as defined above, +the `SealingSegment` structure also stores any resources which are referenced +by blocks in the segment, but not included in the payloads of blocks within +the segment. In particular: +* results referenced by receipts within segment payloads +* results referenced by seals within segment payloads +* seals which represent the latest state commitment as of a segment block + +## Outlook + +In its current state, the sealing segment has been evolving driven by different needs. Most likely, there is some room for simplifications +and other improvements. However, an important aspect of the sealing segment is to allow newly-joining nodes to build an internal representation +of the protocol state, in particular the identity table. There are large changes coming around when we move to the dynamic identity table. +Therefore, we accept that the Sealing Segment currently has some technical debt and unnecessary complexity. Once we have implemented the +dynamic identity table, we will have a much more solidified understanding of the data in the sealing segment. + diff --git a/model/flow/sealing_segment_test.go b/model/flow/sealing_segment_test.go index 00321775074..94034d25b41 100644 --- a/model/flow/sealing_segment_test.go +++ b/model/flow/sealing_segment_test.go @@ -270,7 +270,7 @@ func (suite *SealingSegmentSuite) TestBuild_RootSegment() { unittest.AssertEqualBlocksLenAndOrder(suite.T(), []*flow.Block{root}, segment.Blocks) require.Equal(suite.T(), segment.Highest().ID(), root.ID()) - require.Equal(suite.T(), segment.Lowest().ID(), root.ID()) + require.Equal(suite.T(), segment.Sealed().ID(), root.ID()) } // TestBuild_RootSegmentWrongView tests that we return ErrSegmentInvalidRootView for @@ -422,3 +422,80 @@ func TestAddBlock_StorageError(t *testing.T) { require.ErrorIs(t, err, flow.ErrSegmentSealLookup) }) } + +// TestAddExtraBlock tests different scenarios for adding extra blocks, covers happy and unhappy path scenarios. +func (suite *SealingSegmentSuite) TestAddExtraBlock() { + // populate sealing segment with one block + firstBlock := suite.FirstBlock() + firstBlock.Header.Height += 100 + suite.AddBlocks(firstBlock) + + suite.T().Run("empty-segment", func(t *testing.T) { + builder := flow.NewSealingSegmentBuilder(nil, nil) + block := unittest.BlockFixture() + err := builder.AddExtraBlock(&block) + require.Error(t, err) + }) + suite.T().Run("extra-block-does-not-connect", func(t *testing.T) { + // adding extra block that doesn't connect to the lowest is an error + extraBlock := unittest.BlockFixture() + extraBlock.Header.Height = firstBlock.Header.Height + 10 // make sure it doesn't connect by height + err := suite.builder.AddExtraBlock(&extraBlock) + require.ErrorIs(t, err, flow.ErrSegmentInvalidBlockHeight) + }) + suite.T().Run("extra-block-not-continuous", func(t *testing.T) { + builder := flow.NewSealingSegmentBuilder(suite.GetResult, suite.GetSealByBlockID) + err := builder.AddBlock(firstBlock) + require.NoError(t, err) + extraBlock := unittest.BlockFixture() + extraBlock.Header.Height = firstBlock.Header.Height - 1 // make it connect + err = builder.AddExtraBlock(&extraBlock) + require.NoError(t, err) + extraBlockWithSkip := unittest.BlockFixture() + extraBlockWithSkip.Header.Height = extraBlock.Header.Height - 2 // skip one height + err = builder.AddExtraBlock(&extraBlockWithSkip) + require.ErrorIs(t, err, flow.ErrSegmentInvalidBlockHeight) + }) + suite.T().Run("root-segment-extra-blocks", func(t *testing.T) { + builder := flow.NewSealingSegmentBuilder(suite.GetResult, suite.GetSealByBlockID) + err := builder.AddBlock(firstBlock) + require.NoError(t, err) + + extraBlock := unittest.BlockFixture() + extraBlock.Header.Height = firstBlock.Header.Height - 1 + err = builder.AddExtraBlock(&extraBlock) + require.NoError(t, err) + _, err = builder.SealingSegment() + // root segment cannot have extra blocks + require.Error(t, err) + }) + suite.T().Run("happy-path", func(t *testing.T) { + // add a few blocks with results and seals to form a valid sealing segment + // B1(S*) <- B2(R1) <- B3(S1) + + receipt, seal := unittest.ReceiptAndSealForBlock(firstBlock) + blockWithER := unittest.BlockWithParentFixture(firstBlock.Header) + blockWithER.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt))) + + // add one more block, with seal to the ER + highestBlock := unittest.BlockWithParentFixture(blockWithER.Header) + highestBlock.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) + + suite.AddBlocks(blockWithER, highestBlock) + + // construct two extra blocks that connect to the lowest block and add them to builder + // EB2 <- EB1 <- B1(S*) <- B2(R1) <- B3(S1) + extraBlock := unittest.BlockFixture() + extraBlock.Header.Height = firstBlock.Header.Height - 1 + err := suite.builder.AddExtraBlock(&extraBlock) + require.NoError(t, err) + secondExtraBlock := unittest.BlockFixture() + secondExtraBlock.Header.Height = extraBlock.Header.Height - 1 + err = suite.builder.AddExtraBlock(&secondExtraBlock) + require.NoError(t, err) + segment, err := suite.builder.SealingSegment() + require.NoError(t, err) + err = segment.Validate() + require.NoError(t, err) + }) +} diff --git a/model/flow/timeout_certificate.go b/model/flow/timeout_certificate.go new file mode 100644 index 00000000000..481d86d90b6 --- /dev/null +++ b/model/flow/timeout_certificate.go @@ -0,0 +1,44 @@ +package flow + +import "github.com/onflow/flow-go/crypto" + +// TimeoutCertificate proves that a super-majority of consensus participants want to abandon the specified View. +// At its core, a timeout certificate is an aggregation of TimeoutObjects, which individual nodes send to signal +// their intent to leave the active view. +type TimeoutCertificate struct { + View uint64 + // NewestQCViews lists for each signer (in the same order) the view of the newest QC they supplied + // as part of their TimeoutObject message (specifically TimeoutObject.NewestQC.View). + NewestQCViews []uint64 + // NewestQC is the newest QC from all TimeoutObject that were aggregated for this certificate. + NewestQC *QuorumCertificate + // SignerIndices encodes the HotStuff participants whose TimeoutObjects are included in this TC. + // For `n` authorized consensus nodes, `SignerIndices` is an n-bit vector (padded with tailing + // zeros to reach full bytes). We list the nodes in their canonical order, as defined by the protocol. + SignerIndices []byte + // SigData is an aggregated signature from multiple TimeoutObjects, each from a different replica. + // In their TimeoutObjects, replicas sign the pair (View, NewestQCView) with their staking keys. + SigData crypto.Signature +} + +// ID returns the TimeoutCertificate's identifier +func (t *TimeoutCertificate) ID() Identifier { + if t == nil { + return ZeroID + } + + body := struct { + View uint64 + NewestQCViews []uint64 + NewestQCID Identifier + SignerIndices []byte + SigData crypto.Signature + }{ + View: t.View, + NewestQCViews: t.NewestQCViews, + NewestQCID: t.NewestQC.ID(), + SignerIndices: t.SignerIndices, + SigData: t.SigData, + } + return MakeID(body) +} diff --git a/model/messages/collection.go b/model/messages/collection.go index cd56394a3a7..3ef3251698b 100644 --- a/model/messages/collection.go +++ b/model/messages/collection.go @@ -92,8 +92,8 @@ func NewClusterBlockProposal(internal *cluster.Block) *ClusterBlockProposal { // ClusterBlockVote is a vote for a proposed block in collection node cluster // consensus; effectively a vote for a particular collection. -type ClusterBlockVote struct { - BlockID flow.Identifier - View uint64 - SigData []byte -} +type ClusterBlockVote BlockVote + +// ClusterTimeoutObject is part of the collection cluster protocol and represents a collection node +// timing out in given round. Contains a sequential number for deduplication purposes. +type ClusterTimeoutObject TimeoutObject diff --git a/model/messages/consensus.go b/model/messages/consensus.go index eef83bf91c5..85c2730d1fc 100644 --- a/model/messages/consensus.go +++ b/model/messages/consensus.go @@ -139,3 +139,13 @@ type BlockVote struct { View uint64 SigData []byte } + +// TimeoutObject is part of the consensus protocol and represents a consensus node +// timing out in given round. Contains a sequential number for deduplication purposes. +type TimeoutObject struct { + TimeoutTick uint64 + View uint64 + NewestQC *flow.QuorumCertificate + LastViewTC *flow.TimeoutCertificate + SigData []byte +} diff --git a/module/blobs/blob_store.go b/module/blobs/blob_store.go index 55502929794..903d3358ae9 100644 --- a/module/blobs/blob_store.go +++ b/module/blobs/blob_store.go @@ -81,3 +81,37 @@ func (bs *blobstoreImpl) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error func (bs *blobstoreImpl) HashOnRead(enabled bool) { bs.bs.HashOnRead(enabled) } + +// NoopBlobstore is a Blobstore that does nothing, which is useful for calculating +// BlockExecutionData IDs without storing the data. +type NoopBlobstore struct{} + +func (n *NoopBlobstore) DeleteBlob(context.Context, cid.Cid) error { + return nil +} + +func (n *NoopBlobstore) Has(context.Context, cid.Cid) (bool, error) { + return false, nil +} + +func (n *NoopBlobstore) Get(context.Context, cid.Cid) (Blob, error) { + return nil, nil +} + +func (n *NoopBlobstore) GetSize(context.Context, cid.Cid) (int, error) { + return 0, nil +} + +func (n *NoopBlobstore) Put(context.Context, Blob) error { + return nil +} + +func (n *NoopBlobstore) PutMany(context.Context, []Blob) error { + return nil +} + +func (n *NoopBlobstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, nil +} + +func (n *NoopBlobstore) HashOnRead(enabled bool) {} diff --git a/module/buffer.go b/module/buffer.go index 91291d487f6..d8cf73ea6b4 100644 --- a/module/buffer.go +++ b/module/buffer.go @@ -9,6 +9,7 @@ import ( // cannot yet be processed because they do not connect to the rest of the chain // state. They are indexed by parent ID to enable processing all of a parent's // children once the parent is received. +// Safe for concurrent use. type PendingBlockBuffer interface { Add(originID flow.Identifier, block *flow.Block) bool @@ -26,6 +27,7 @@ type PendingBlockBuffer interface { // PendingClusterBlockBuffer is the same thing as PendingBlockBuffer, but for // collection node cluster consensus. +// Safe for concurrent use. type PendingClusterBlockBuffer interface { Add(originID flow.Identifier, block *cluster.Block) bool diff --git a/module/builder/collection/builder.go b/module/builder/collection/builder.go index 5841db310db..4a65e194279 100644 --- a/module/builder/collection/builder.go +++ b/module/builder/collection/builder.go @@ -147,7 +147,6 @@ func (b *Builder) BuildOn(parentID flow.Identifier, setter func(*flow.Header) er Str("chain_id", parent.ChainID.String()). Uint64("final_ref_height", refChainFinalizedHeight). Logger() - log.Debug().Msg("building new cluster block") // TODO (ramtin): enable this again diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 6abdd569d53..1bdcff76392 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -79,7 +79,8 @@ func (suite *BuilderSuite) SetupTest() { suite.blocks = blocks suite.payloads = storage.NewClusterPayloads(metrics, suite.db) - clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis) + clusterQC := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) + clusterStateRoot, err := clusterkv.NewStateRoot(suite.genesis, clusterQC) suite.Require().NoError(err) clusterState, err := clusterkv.Bootstrap(suite.db, clusterStateRoot) suite.Require().NoError(err) @@ -90,12 +91,11 @@ func (suite *BuilderSuite) SetupTest() { // just bootstrap with a genesis block, we'll use this as reference participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) root, result, seal := unittest.BootstrapFixture(participants) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(root.ID())) // ensure we don't enter a new epoch for tests that build many blocks result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = root.Header.View + 100000 seal.ResultID = result.ID() - rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) + rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID()))) require.NoError(suite.T(), err) state, err := pbadger.Bootstrap(metrics, suite.db, headers, seals, results, blocks, setups, commits, statuses, rootSnapshot) @@ -983,7 +983,8 @@ func benchmarkBuildOn(b *testing.B, size int) { suite.blocks = blocks suite.payloads = storage.NewClusterPayloads(metrics, suite.db) - stateRoot, err := clusterkv.NewStateRoot(suite.genesis) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(suite.genesis.ID())) + stateRoot, err := clusterkv.NewStateRoot(suite.genesis, qc) state, err := clusterkv.Bootstrap(suite.db, stateRoot) assert.NoError(b, err) diff --git a/module/builder/consensus/builder.go b/module/builder/consensus/builder.go index 67b29a3aaa3..ef78d8751f9 100644 --- a/module/builder/consensus/builder.go +++ b/module/builder/consensus/builder.go @@ -223,7 +223,7 @@ func (b *Builder) repopulateExecutionTree() error { // At this point execution tree is filled with all results for blocks (lastSealedBlock, lastFinalizedBlock]. // Now, we add all known receipts for any valid block that descends from the latest finalized block: - validPending, err := finalizedSnapshot.ValidDescendants() + validPending, err := finalizedSnapshot.Descendants() if err != nil { return fmt.Errorf("could not retrieve valid pending blocks from finalized snapshot: %w", err) } diff --git a/module/builder/consensus/builder_test.go b/module/builder/consensus/builder_test.go index 77c4b243437..9fe8a563fee 100644 --- a/module/builder/consensus/builder_test.go +++ b/module/builder/consensus/builder_test.go @@ -275,7 +275,7 @@ func (bs *BuilderSuite) SetupTest() { bs.state.On("Final").Return(func() realproto.Snapshot { if block, ok := bs.blocks[bs.finalID]; ok { snapshot := unittest.StateSnapshotForKnownBlock(block.Header, nil) - snapshot.On("ValidDescendants").Return(bs.blockChildren[bs.finalID], nil) + snapshot.On("Descendants").Return(bs.blockChildren[bs.finalID], nil) return snapshot } return unittest.StateSnapshotForUnknownBlock() @@ -595,8 +595,7 @@ func (bs *BuilderSuite) TestPayloadSeals_OnlyFork() { // // SCENARIO: // - block B0 is sealed -// Proposer for ░newBlock░: -// - Knows block A5. Hence, it knows a QC for block B4, which contains the Source Of Randomness (SOR) for B4. +// Proposer for ░newBlock░ knows block A5. Hence, it knows a QC for block B4, which contains the Source Of Randomness (SOR) for B4. // Therefore, the proposer can construct the verifier assignment for [B4{incorporates result R for B1}] // - Assume that verification was fast enough, so the proposer has sufficient approvals for result R. // Therefore, the proposer has a candidate seal, sealing result R for block B4, in its mempool. @@ -605,8 +604,7 @@ func (bs *BuilderSuite) TestPayloadSeals_OnlyFork() { // // - Assume that the replica does _not_ know A5. Therefore, it _cannot_ compute the verifier assignment for B4. // -// Problem: If the proposer included the seal for B1, the replica could not check it. -// +// Problem: If the proposer included the seal for B1, the replica could not check it. // Solution: There must be a gap between the block incorporating the result (here B4) and // the block sealing the result. A gap of one block is sufficient. // diff --git a/module/chainsync/core.go b/module/chainsync/core.go index 449f40b2e3a..08514b1b513 100644 --- a/module/chainsync/core.go +++ b/module/chainsync/core.go @@ -1,6 +1,7 @@ package chainsync import ( + "fmt" "sort" "sync" "time" @@ -62,9 +63,9 @@ type Core struct { localFinalizedHeight uint64 } -func New(log zerolog.Logger, config Config, metrics module.ChainSyncMetrics) (*Core, error) { +func New(log zerolog.Logger, config Config, metrics module.ChainSyncMetrics, chainID flow.ChainID) (*Core, error) { core := &Core{ - log: log.With().Str("module", "synchronization").Logger(), + log: log.With().Str("sync_core", chainID.String()).Logger(), Config: config, heights: make(map[uint64]*chainsync.Status), blockIDs: make(map[flow.Identifier]*chainsync.Status), @@ -78,16 +79,23 @@ func New(log zerolog.Logger, config Config, metrics module.ChainSyncMetrics) (*C // true if the block should be processed by the compliance layer and false // if it should be ignored. func (c *Core) HandleBlock(header *flow.Header) bool { + log := c.log + if c.log.Debug().Enabled() { + log = c.log.With().Str("block_id", header.ID().String()).Uint64("block_height", header.Height).Logger() + } c.mu.Lock() defer c.mu.Unlock() status := c.getRequestStatus(header.Height, header.ID()) + // if we never asked for this block, discard it if !status.WasQueued() { + log.Debug().Msg("discarding not queued block") return false } // if we have already received this block, exit if status.WasReceived() { + log.Debug().Msg("discarding not received block") return false } @@ -99,6 +107,7 @@ func (c *Core) HandleBlock(header *flow.Header) bool { c.blockIDs[header.ID()] = status c.heights[header.Height] = status + log.Debug().Msg("handled block") return true } @@ -106,8 +115,11 @@ func (c *Core) HandleBlock(header *flow.Header) bool { // If the height difference between local and the reported height, we do nothing. // Otherwise, we queue each missing height. func (c *Core) HandleHeight(final *flow.Header, height uint64) { + log := c.log.With().Uint64("final_height", final.Height).Uint64("recv_height", height).Logger() + log.Debug().Msg("received height") // don't bother queueing anything if we're within tolerance if c.WithinTolerance(final, height) { + log.Debug().Msg("height within tolerance - discarding") return } @@ -127,10 +139,12 @@ func (c *Core) HandleHeight(final *flow.Header, height uint64) { for h := final.Height + 1; h <= height; h++ { c.requeueHeight(h) } + log.Debug().Msgf("requeued heights [%d-%d]", final.Height+1, height) } } func (c *Core) RequestBlock(blockID flow.Identifier, height uint64) { + log := c.log.With().Str("block_id", blockID.String()).Uint64("height", height).Logger() // requesting a block by its ID storing the height to prune more efficiently c.mu.Lock() defer c.mu.Unlock() @@ -138,11 +152,13 @@ func (c *Core) RequestBlock(blockID flow.Identifier, height uint64) { // if we already received this block, reset the status so we can re-queue status := c.blockIDs[blockID] if status.WasReceived() { + log.Debug().Msgf("requested block was already received") delete(c.blockIDs, status.Header.ID()) delete(c.heights, status.Header.Height) } c.queueByBlockID(blockID, height) + log.Debug().Msgf("enqueued requested block") } func (c *Core) RequestHeight(height uint64) { @@ -150,6 +166,7 @@ func (c *Core) RequestHeight(height uint64) { defer c.mu.Unlock() c.requeueHeight(height) + c.log.Debug().Uint64("height", height).Msg("enqueued requested height") } // requeueHeight queues the given height, ignoring any previously received @@ -172,15 +189,19 @@ func (c *Core) ScanPending(final *flow.Header) ([]chainsync.Range, []chainsync.B c.mu.Lock() defer c.mu.Unlock() + log := c.log.With().Uint64("final_height", final.Height).Logger() + // prune if the current height is less than the new height c.prune(final) // get all items that are eligible for initial or re-requesting heights, blockIDs := c.getRequestableItems() + c.log.Debug().Msgf("scan found %d requestable heights, %d requestable block IDs", len(heights), len(blockIDs)) // convert to valid range and batch requests ranges := c.getRanges(heights) batches := c.getBatches(blockIDs) + log.Debug().Str("ranges", fmt.Sprintf("%v", ranges)).Str("batches", fmt.Sprintf("%v", batches)).Msg("compiled range and batch requests") return c.selectRequests(ranges, batches) } diff --git a/module/chainsync/core_rapid_test.go b/module/chainsync/core_rapid_test.go index f98bd32bdcd..649fce871d8 100644 --- a/module/chainsync/core_rapid_test.go +++ b/module/chainsync/core_rapid_test.go @@ -42,7 +42,7 @@ type rapidSync struct { func (r *rapidSync) Init(t *rapid.T) { var err error - r.core, err = New(zerolog.New(io.Discard), DefaultConfig(), metrics.NewNoopCollector()) + r.core, err = New(zerolog.New(io.Discard), DefaultConfig(), metrics.NewNoopCollector(), flow.Localnet) require.NoError(t, err) r.store = populatedBlockStore(t) diff --git a/module/chainsync/core_test.go b/module/chainsync/core_test.go index 3d563714289..d8feca7e2fd 100644 --- a/module/chainsync/core_test.go +++ b/module/chainsync/core_test.go @@ -30,7 +30,7 @@ type SyncSuite struct { func (ss *SyncSuite) SetupTest() { var err error - ss.core, err = New(zerolog.New(io.Discard), DefaultConfig(), metrics.NewNoopCollector()) + ss.core, err = New(zerolog.New(io.Discard), DefaultConfig(), metrics.NewNoopCollector(), flow.Localnet) ss.Require().Nil(err) } diff --git a/module/chunks/chunkVerifier.go b/module/chunks/chunkVerifier.go index 5b3ae2e47e2..f72a3148ef0 100644 --- a/module/chunks/chunkVerifier.go +++ b/module/chunks/chunkVerifier.go @@ -7,7 +7,6 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/fvm/blueprints" - "github.com/onflow/flow-go/model/convert" "github.com/onflow/flow-go/model/verification" "github.com/onflow/flow-go/engine/execution/computation/computer" @@ -92,6 +91,34 @@ func (fcv *ChunkVerifier) Verify( vc.IsSystemChunk) } +type partialLedgerStorageSnapshot struct { + snapshot delta.StorageSnapshot + + unknownRegTouch map[flow.RegisterID]struct{} +} + +func (storage *partialLedgerStorageSnapshot) Get( + id flow.RegisterID, +) ( + flow.RegisterValue, + error, +) { + value, err := storage.snapshot.Get(id) + if err != nil && errors.Is(err, ledger.ErrMissingKeys{}) { + storage.unknownRegTouch[id] = struct{}{} + + // don't send error just return empty byte slice + // we always assume empty value for missing registers (which might + // cause the transaction to fail) + // but after execution we check unknownRegTouch and if any + // register is inside it, code won't generate approvals and + // it activates a challenge + return flow.RegisterValue{}, nil + } + + return value, err +} + func (fcv *ChunkVerifier) verifyTransactionsInContext( context fvm.Context, transactionOffset uint32, @@ -119,14 +146,18 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( } events := make(flow.EventsList, 0) - serviceEvents := make(flow.EventsList, 0) + serviceEvents := make(flow.ServiceEventList, 0) // constructing a partial trie given chunk data package psmt, err := partial.NewLedger(chunkDataPack.Proof, ledger.State(chunkDataPack.StartState), partial.DefaultPathFinderVersion) if err != nil { // TODO provide more details based on the error type - return nil, chmodels.NewCFInvalidVerifiableChunk("error constructing partial trie: ", err, chIndex, execResID), + return nil, chmodels.NewCFInvalidVerifiableChunk( + "error constructing partial trie: ", + err, + chIndex, + execResID), nil } @@ -139,44 +170,16 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( // chunk view construction // unknown register tracks access to parts of the partial trie which // are not expanded and values are unknown. - unknownRegTouch := make(map[flow.RegisterID]*ledger.Key) - var problematicTx flow.Identifier - getRegister := func(owner, key string) (flow.RegisterValue, error) { - // check if register has been provided in the chunk data pack - registerID := flow.NewRegisterID(owner, key) - - registerKey := executionState.RegisterIDToKey(registerID) - - query, err := ledger.NewQuerySingleValue(ledger.State(chunkDataPack.StartState), registerKey) - - if err != nil { - return nil, fmt.Errorf("cannot create query: %w", err) - } - - value, err := psmt.GetSingleValue(query) - if err != nil { - if errors.Is(err, ledger.ErrMissingKeys{}) { - - unknownRegTouch[registerID] = ®isterKey - - // don't send error just return empty byte slice - // we always assume empty value for missing registers (which might cause the transaction to fail) - // but after execution we check unknownRegTouch and if any - // register is inside it, code won't generate approvals and - // it activates a challenge - - return []byte{}, nil - } - // append to missing keys if error is ErrMissingKeys - - return nil, fmt.Errorf("cannot query register: %w", err) - } - - return value, nil - } - - chunkView := delta.NewView(getRegister) + unknownRegTouch := make(map[flow.RegisterID]struct{}) + chunkView := delta.NewDeltaView( + &partialLedgerStorageSnapshot{ + snapshot: executionState.NewLedgerStorageSnapshot( + psmt, + chunkDataPack.StartState), + unknownRegTouch: unknownRegTouch, + }) + var problematicTx flow.Identifier // executes all transactions in this chunk for i, tx := range transactions { txView := chunkView.NewChild() @@ -193,7 +196,7 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( } events = append(events, tx.Events...) - serviceEvents = append(serviceEvents, tx.ServiceEvents...) + serviceEvents = append(serviceEvents, tx.ConvertedServiceEvents...) // always merge back the tx view (fvm is responsible for changes on tx errors) err = chunkView.MergeView(txView) @@ -205,8 +208,8 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( // check read access to unknown registers if len(unknownRegTouch) > 0 { var missingRegs []string - for _, key := range unknownRegTouch { - missingRegs = append(missingRegs, key.String()) + for id := range unknownRegTouch { + missingRegs = append(missingRegs, id.String()) } return nil, chmodels.NewCFMissingRegisterTouch(missingRegs, chIndex, execResID, problematicTx), nil } @@ -238,23 +241,12 @@ func (fcv *ChunkVerifier) verifyTransactionsInContext( } if systemChunk { - - computedServiceEvents := make(flow.ServiceEventList, len(serviceEvents)) - - for i, serviceEvent := range serviceEvents { - realServiceEvent, err := convert.ServiceEvent(fcv.vmCtx.Chain.ChainID(), serviceEvent) - if err != nil { - return nil, nil, fmt.Errorf("cannot convert service event %d: %w", i, err) - } - computedServiceEvents[i] = *realServiceEvent - } - - equal, err := result.ServiceEvents.EqualTo(computedServiceEvents) + equal, err := result.ServiceEvents.EqualTo(serviceEvents) if err != nil { - return nil, nil, fmt.Errorf("error while compariong service events: %w", err) + return nil, nil, fmt.Errorf("error while comparing service events: %w", err) } if !equal { - return nil, chmodels.CFInvalidServiceSystemEventsEmitted(result.ServiceEvents, computedServiceEvents, chIndex, execResID), nil + return nil, chmodels.CFInvalidServiceSystemEventsEmitted(result.ServiceEvents, serviceEvents, chIndex, execResID), nil } } diff --git a/module/chunks/chunkVerifier_test.go b/module/chunks/chunkVerifier_test.go index 72a7bed3f58..98fe3afca61 100644 --- a/module/chunks/chunkVerifier_test.go +++ b/module/chunks/chunkVerifier_test.go @@ -52,6 +52,7 @@ var epochSetupEvent, _ = convertfixtures.EpochSetupFixtureByChainID(testChain) var epochCommitEvent, _ = convertfixtures.EpochCommitFixtureByChainID(testChain) var epochSetupServiceEvent, _ = convert.ServiceEvent(testChain, epochSetupEvent) +var epochCommitServiceEvent, _ = convert.ServiceEvent(testChain, epochCommitEvent) var serviceEventsList = []flow.ServiceEvent{ *epochSetupServiceEvent, @@ -360,15 +361,18 @@ func (vm *vmMock) Run(ctx fvm.Context, proc fvm.Procedure, led state.View) error return fmt.Errorf("invokable is not a transaction") } + id0 := flow.NewRegisterID("00", "") + id5 := flow.NewRegisterID("05", "") + switch string(tx.Transaction.Script) { case "wrongEndState": // add updates to the ledger - _ = led.Set("00", "", []byte{'F'}) + _ = led.Set(id0, []byte{'F'}) tx.Logs = []string{"log1", "log2"} tx.Events = eventsList case "failedTx": // add updates to the ledger - _ = led.Set("05", "", []byte{'B'}) + _ = led.Set(id5, []byte{'B'}) tx.Err = fvmErrors.NewCadenceRuntimeError(runtime.Error{}) // inside the runtime (e.g. div by zero, access account) case "eventsMismatch": tx.Events = append(eventsList, flow.Event{ @@ -379,9 +383,9 @@ func (vm *vmMock) Run(ctx fvm.Context, proc fvm.Procedure, led state.View) error Payload: []byte{88}, }) default: - _, _ = led.Get("00", "") - _, _ = led.Get("05", "") - _ = led.Set("05", "", []byte{'B'}) + _, _ = led.Get(id0) + _, _ = led.Get(id5) + _ = led.Set(id5, []byte{'B'}) tx.Logs = []string{"log1", "log2"} tx.Events = eventsList } @@ -401,12 +405,15 @@ func (vm *vmSystemOkMock) Run(ctx fvm.Context, proc fvm.Procedure, led state.Vie return fmt.Errorf("invokable is not a transaction") } - tx.ServiceEvents = []flow.Event{epochSetupEvent} + tx.ConvertedServiceEvents = flow.ServiceEventList{*epochSetupServiceEvent} + + id0 := flow.NewRegisterID("00", "") + id5 := flow.NewRegisterID("05", "") // add "default" interaction expected in tests - _, _ = led.Get("00", "") - _, _ = led.Get("05", "") - _ = led.Set("05", "", []byte{'B'}) + _, _ = led.Get(id0) + _, _ = led.Get(id5) + _ = led.Set(id5, []byte{'B'}) tx.Logs = []string{"log1", "log2"} return nil @@ -424,7 +431,7 @@ func (vm *vmSystemBadMock) Run(ctx fvm.Context, proc fvm.Procedure, led state.Vi return fmt.Errorf("invokable is not a transaction") } // EpochSetup event is expected, but we emit EpochCommit here resulting in a chunk fault - tx.ServiceEvents = []flow.Event{epochCommitEvent} + tx.ConvertedServiceEvents = flow.ServiceEventList{*epochCommitServiceEvent} return nil } diff --git a/module/component/component.go b/module/component/component.go index 2c0a7c39c2b..34f8f61cf14 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -132,6 +132,13 @@ type ReadyFunc func() // waits until all workers have signaled that they are ready before closing its own Ready channel. type ComponentWorker func(ctx irrecoverable.SignalerContext, ready ReadyFunc) +// NoopWorker is a worker routine which is immediately ready, does nothing, and +// exits when the context is done. +func NoopWorker(ctx irrecoverable.SignalerContext, ready ReadyFunc) { + ready() + <-ctx.Done() +} + // ComponentManagerBuilder provides a mechanism for building a ComponentManager type ComponentManagerBuilder interface { // AddWorker adds a worker routine for the ComponentManager diff --git a/module/epochs.go b/module/epochs.go index 4e73dad870e..874fc44a4fc 100644 --- a/module/epochs.go +++ b/module/epochs.go @@ -12,12 +12,12 @@ import ( // next epoch. type ClusterRootQCVoter interface { - // Vote handles the full procedure of generating a vote, submitting it to the - // epoch smart contract, and verifying submission. Returns an error only if - // there is a critical error that would make it impossible for the vote to be - // submitted. Otherwise, exits when the vote has been successfully submitted. - // - // It is safe to run Vote multiple times within a single setup phase. + // Vote handles the full procedure of generating a vote, submitting it to the epoch + // smart contract, and verifying submission. It is safe to run Vote multiple + // times within a single setup phase. + // Error returns: + // - epochs.ClusterQCNoVoteError if we fail to vote for a benign reason + // - generic error in case of critical unexpected failure Vote(context.Context, protocol.Epoch) error } @@ -30,19 +30,31 @@ type QCContractClient interface { // contract. This function returns only once the transaction has been // processed by the network. An error is returned if the transaction has // failed and should be re-submitted. + // Error returns: + // - network.TransientError for any errors from the underlying client, if the retry period has been exceeded + // - errTransactionExpired if the transaction has expired + // - errTransactionReverted if the transaction execution reverted + // - generic error in case of unexpected critical failure SubmitVote(ctx context.Context, vote *model.Vote) error // Voted returns true if we have successfully submitted a vote to the // cluster QC aggregator smart contract for the current epoch. + // Error returns: + // - network.TransientError for any errors from the underlying Flow client + // - generic error in case of unexpected critical failures Voted(ctx context.Context) (bool, error) } -// EpochLookup provides a method to find epochs by view. +// EpochLookup enables looking up epochs by view. type EpochLookup interface { - // EpochForView returns the counter of the epoch that the view belongs to. - EpochForView(view uint64) (epochCounter uint64, err error) - - // EpochForViewWithFallback returns the counter of the epoch that the view belongs to. + // EpochForViewWithFallback returns the counter of the epoch that the input view belongs to. + // If epoch fallback has been triggered, returns the last committed epoch counter + // in perpetuity for any inputs beyond the last committed epoch view range. + // For example, if we trigger epoch fallback during epoch 10, and reach the final + // view of epoch 10 before epoch 11 has finished being setup, this function will + // return 10 even for input views beyond the final view of epoch 10. + // + // Returns model.ErrViewForUnknownEpoch if the input does not fall within the range of a known epoch. EpochForViewWithFallback(view uint64) (epochCounter uint64, err error) } diff --git a/module/epochs/base_client.go b/module/epochs/base_client.go index dcc7bc91c07..eea76558639 100644 --- a/module/epochs/base_client.go +++ b/module/epochs/base_client.go @@ -2,6 +2,7 @@ package epochs import ( "context" + "errors" "fmt" "time" @@ -11,6 +12,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/module" ) @@ -20,6 +22,13 @@ const ( waitForSealedMaxDuration = 5 * time.Minute ) +var ( + // errTransactionExpired is returned when a transaction expires before it is incorporated into a block. + errTransactionExpired = errors.New("transaction expired") + // errTransactionReverted is returned when a transaction is executed, but the execution reverts. + errTransactionReverted = errors.New("transaction execution reverted") +) + // BaseClient represents the core fields and methods needed to create // a client to a contract on the Flow Network. type BaseClient struct { @@ -53,23 +62,32 @@ func NewBaseClient( } } +// GetAccount returns the current state for the account associated with the BaseClient. +// Error returns: +// - network.TransientError for any errors from the underlying client +// - generic error in case of unexpected critical failure func (c *BaseClient) GetAccount(ctx context.Context) (*sdk.Account, error) { // get account from access node for given address account, err := c.FlowClient.GetAccount(ctx, c.AccountAddress) if err != nil { - return nil, fmt.Errorf("could not get account: %w", err) + // we consider all errors from client network calls to be transient and non-critical + return nil, network.NewTransientErrorf("could not get account: %w", err) } // check if account key index within range of keys - if len(account.Keys) <= int(c.AccountKeyIndex) { - return nil, fmt.Errorf("given account key index is bigger than the number of keys for this account") + if int(c.AccountKeyIndex) >= len(account.Keys) { + return nil, fmt.Errorf("given account key index exceeds the number of keys for this account (%d>=%d)", + c.AccountKeyIndex, len(account.Keys)) } return account, nil } // SendTransaction submits a transaction to Flow. Requires transaction to be signed. +// Error returns: +// - network.TransientError for any errors from the underlying client +// - generic error in case of unexpected critical failure func (c *BaseClient) SendTransaction(ctx context.Context, tx *sdk.Transaction) (sdk.Identifier, error) { // check if the transaction has a signature @@ -80,13 +98,19 @@ func (c *BaseClient) SendTransaction(ctx context.Context, tx *sdk.Transaction) ( // submit transaction to client err := c.FlowClient.SendTransaction(ctx, *tx) if err != nil { - return sdk.EmptyID, fmt.Errorf("failed to send transaction: %w", err) + // we consider all errors from client network calls to be transient and non-critical + return sdk.EmptyID, network.NewTransientErrorf("failed to send transaction: %w", err) } return tx.ID(), nil } // WaitForSealed waits for a transaction to be sealed +// Error returns: +// - network.TransientError for any errors from the underlying client, if the retry period has been exceeded +// - errTransactionExpired if the transaction has expired +// - errTransactionReverted if the transaction execution reverted +// - generic error in case of unexpected critical failure func (c *BaseClient) WaitForSealed(ctx context.Context, txID sdk.Identifier, started time.Time) error { log := c.Log.With().Str("tx_id", txID.Hex()).Logger() @@ -101,30 +125,31 @@ func (c *BaseClient) WaitForSealed(ctx context.Context, txID sdk.Identifier, sta result, err := c.FlowClient.GetTransactionResult(ctx, txID) if err != nil { - msg := "could not get transaction result, retrying" - log.Error().Err(err).Msg(msg) - return retry.RetryableError(fmt.Errorf(msg)) + // we consider all errors from client network calls to be transient and non-critical + err = network.NewTransientErrorf("could not get transaction result: %w", err) + log.Err(err).Msg("retrying getting transaction result...") + return retry.RetryableError(err) } if result.Error != nil { - return fmt.Errorf("error executing transaction: %w", result.Error) + return fmt.Errorf("transaction reverted with error=[%s]: %w", result.Error.Error(), errTransactionReverted) } log.Info().Str("status", result.Status.String()).Msg("got transaction result") // if the transaction has expired we skip waiting for seal if result.Status == sdk.TransactionStatusExpired { - return fmt.Errorf("transaction has expired") + return errTransactionExpired } if result.Status == sdk.TransactionStatusSealed { return nil } - return retry.RetryableError(fmt.Errorf("waiting for transaction to be sealed retrying")) + return retry.RetryableError(network.NewTransientErrorf("transaction not sealed yet (status=%s)", result.Status)) }) if err != nil { - return err + return fmt.Errorf("transaction (id=%s) failed to be sealed successfully after %s: %w", txID.String(), time.Since(started), err) } return nil diff --git a/module/epochs/epoch_lookup.go b/module/epochs/epoch_lookup.go index 2bd42cdc380..f0ec869a0ec 100644 --- a/module/epochs/epoch_lookup.go +++ b/module/epochs/epoch_lookup.go @@ -1,101 +1,281 @@ package epochs import ( - "errors" "fmt" + "sync" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/events" ) -// EpochLookup implements the EpochLookup interface using protocol state to -// match views to epochs. +// epochRange captures the counter and view range of an epoch (inclusive on both ends) +type epochRange struct { + counter uint64 + firstView uint64 + finalView uint64 +} + +// exists returns true when the epochRange is initialized (anything besides the zero value for the struct). +// It is useful for checking existence while iterating the epochRangeCache. +func (er epochRange) exists() bool { + return er != epochRange{} +} + +// epochRangeCache stores at most the 3 latest epoch ranges. +// Ranges are ordered by counter (ascending) and right-aligned. +// For example, if we only have one epoch cached, `epochRangeCache[0]` and `epochRangeCache[1]` are `nil`. +// Not safe for concurrent use. +type epochRangeCache [3]epochRange + +// latest returns the latest cached epoch range, or nil if no epochs are cached. +func (cache *epochRangeCache) latest() epochRange { + return cache[2] +} + +// combinedRange returns the endpoints of the combined view range of all cached +// epochs. In particular, we return the lowest firstView and the greatest finalView. +// At least one epoch must already be cached, otherwise this function will panic. +func (cache *epochRangeCache) combinedRange() (firstView uint64, finalView uint64) { + + // low end of the range is the first view of the first cached epoch + for _, epoch := range cache { + if epoch.exists() { + firstView = epoch.firstView + break + } + } + // high end of the range is the final view of the latest cached epoch + finalView = cache.latest().finalView + return +} + +// add inserts an epoch range to the cache. +// Validates that epoch counters and view ranges are sequential. +// Adding the same epoch multiple times is a no-op. +// Guarantees ordering and alignment properties of epochRangeCache are preserved. +// No errors are expected during normal operation. +func (cache *epochRangeCache) add(epoch epochRange) error { + + // sanity check: ensure the epoch we are adding is considered a non-zero value + // this helps ensure internal consistency in this component, but if we ever trip this check, something is seriously wrong elsewhere + if !epoch.exists() { + return fmt.Errorf("sanity check failed: caller attempted to cache invalid zero epoch") + } + + latestCachedEpoch := cache.latest() + // initial case - no epoch ranges are stored yet + if !latestCachedEpoch.exists() { + cache[2] = epoch + return nil + } + + // adding the same epoch multiple times is a no-op + if latestCachedEpoch == epoch { + return nil + } + + // sanity check: ensure counters/views are sequential + if epoch.counter != latestCachedEpoch.counter+1 { + return fmt.Errorf("non-sequential epoch counters: adding epoch %d when latest cached epoch is %d", epoch.counter, latestCachedEpoch.counter) + } + if epoch.firstView != latestCachedEpoch.finalView+1 { + return fmt.Errorf("non-sequential epoch view ranges: adding range [%d,%d] when latest cached range is [%d,%d]", + epoch.firstView, epoch.finalView, latestCachedEpoch.firstView, latestCachedEpoch.finalView) + } + + // typical case - displacing existing epoch ranges + // insert new epoch range, shifting existing epochs left + cache[0] = cache[1] // ejects oldest epoch + cache[1] = cache[2] + cache[2] = epoch + + return nil +} + +// EpochLookup implements the EpochLookup interface using protocol state to match views to epochs. type EpochLookup struct { - state protocol.State + state protocol.State + mu sync.RWMutex + epochs epochRangeCache + committedEpochsCh chan *flow.Header // protocol events for newly committed epochs (the first block of the epoch is passed over the channel) + epochFallbackIsTriggered *atomic.Bool // true when epoch fallback is triggered + events.Noop // implements protocol.Consumer + component.Component } +var _ protocol.Consumer = (*EpochLookup)(nil) +var _ module.EpochLookup = (*EpochLookup)(nil) + // NewEpochLookup instantiates a new EpochLookup -func NewEpochLookup(state protocol.State) *EpochLookup { - return &EpochLookup{ - state: state, +func NewEpochLookup(state protocol.State) (*EpochLookup, error) { + lookup := &EpochLookup{ + state: state, + committedEpochsCh: make(chan *flow.Header, 1), + epochFallbackIsTriggered: atomic.NewBool(false), } -} -// EpochForView returns the counter of the epoch that the view belongs to. -// The protocol.State#Epochs object exposes previous, current, and next epochs, -// which should be all we need. In general, we can't guarantee that a node will -// have access to epoch data beyond these three, so it is safe to throw an error -// for a query that doesn't fit within the view bounds of these three epochs -// (even if the node does happen to have that stored in the underlying storage) -// -- these queries indicate a bug in the querier. -func (l *EpochLookup) EpochForView(view uint64) (epochCounter uint64, err error) { - epochs := l.state.Final().Epochs() - previous := epochs.Previous() - current := epochs.Current() - next := epochs.Next() - - for _, epoch := range []protocol.Epoch{previous, current, next} { - counter, err := epoch.Counter() - if errors.Is(err, protocol.ErrNoPreviousEpoch) { - continue - } - if err != nil { - return 0, err - } + lookup.Component = component.NewComponentManagerBuilder(). + AddWorker(lookup.handleProtocolEvents). + Build() - firstView, err := epoch.FirstView() + final := state.Final() + + // we cache the previous epoch, if one exists + exists, err := protocol.PreviousEpochExists(final) + if err != nil { + return nil, fmt.Errorf("could not check previous epoch exists: %w", err) + } + if exists { + err := lookup.cacheEpoch(final.Epochs().Previous()) if err != nil { - return 0, err + return nil, fmt.Errorf("could not prepare previous epoch: %w", err) } - finalView, err := epoch.FinalView() + } + + // we always cache the current epoch + err = lookup.cacheEpoch(final.Epochs().Current()) + if err != nil { + return nil, fmt.Errorf("could not prepare current epoch: %w", err) + } + + // we cache the next epoch, if it is committed + phase, err := final.Phase() + if err != nil { + return nil, fmt.Errorf("could not check epoch phase: %w", err) + } + if phase == flow.EpochPhaseCommitted { + err := lookup.cacheEpoch(final.Epochs().Next()) if err != nil { - return 0, err - } - if firstView <= view && view <= finalView { - return counter, nil + return nil, fmt.Errorf("could not prepare previous epoch: %w", err) } } - return 0, fmt.Errorf("couldn't get epoch for view %d", view) + // if epoch fallback was triggered, note it here + triggered, err := state.Params().EpochFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not check epoch fallback: %w", err) + } + if triggered { + lookup.epochFallbackIsTriggered.Store(true) + } + + return lookup, nil } -// EpochForViewWithFallback returns the counter of the epoch that the input -// view belongs to, with the same rules as EpochForView, except that this -// function will return the last committed epoch counter in perpetuity in the -// case that any epoch preparation. For example, if we are in epoch 10, and -// reach the final view of epoch 10 before epoch 11 has finished being setup, -// this function will return 10 even after the final view of epoch 10. -func (l *EpochLookup) EpochForViewWithFallback(view uint64) (uint64, error) { - - epochs := l.state.Final().Epochs() - current := epochs.Current() - next := epochs.Next() - - // TMP: EMERGENCY EPOCH CHAIN CONTINUATION [EECC] - // - // If the given view is within the bounds of the next epoch, and the epoch - // has not been set up or committed, we pretend that we are still in the - // current epoch and return that epoch's counter. - // - // This is used to determine which Random Beacon key we will use to sign and - // verify blocks and votes. The failure case we are avoiding here is if the - // DKG for next epoch failed and there is no Random Beacon key for that epoch, - // or if the next epoch failed for any other reason. In either case we will - // continue using the last valid Random Beacon key until the next spork. - // - currentFinalView, err := current.FinalView() +// cacheEpoch caches the given epoch's view range. Must only be called with committed epochs. +// No errors are expected during normal operation. +func (lookup *EpochLookup) cacheEpoch(epoch protocol.Epoch) error { + counter, err := epoch.Counter() if err != nil { - return 0, err + return err } - if view > currentFinalView { - _, err := next.DKG() // either of the following errors indicates that we have transitioned into EECC - if errors.Is(err, protocol.ErrEpochNotCommitted) || errors.Is(err, protocol.ErrNextEpochNotSetup) { - return current.Counter() + firstView, err := epoch.FirstView() + if err != nil { + return err + } + finalView, err := epoch.FinalView() + if err != nil { + return err + } + + cachedEpoch := epochRange{ + counter: counter, + firstView: firstView, + finalView: finalView, + } + + lookup.mu.Lock() + err = lookup.epochs.add(cachedEpoch) + lookup.mu.Unlock() + if err != nil { + return fmt.Errorf("could not add epoch %d: %w", counter, err) + } + return nil +} + +// EpochForViewWithFallback returns the counter of the epoch that the input view belongs to. +// If epoch fallback has been triggered, returns the last committed epoch counter +// in perpetuity for any inputs beyond the last committed epoch view range. +// For example, if we trigger epoch fallback during epoch 10, and reach the final +// view of epoch 10 before epoch 11 has finished being setup, this function will +// return 10 even for input views beyond the final view of epoch 10. +// +// Returns model.ErrViewForUnknownEpoch if the input does not fall within the range of a known epoch. +func (lookup *EpochLookup) EpochForViewWithFallback(view uint64) (uint64, error) { + lookup.mu.RLock() + defer lookup.mu.RUnlock() + firstView, finalView := lookup.epochs.combinedRange() + + // LEGEND: + // * -> view argument + // [----| -> epoch view range + + // view is before any known epochs + // ---*---[----|----|----]------- + if view < firstView { + return 0, model.ErrViewForUnknownEpoch + } + // view is after any known epochs + // -------[----|----|----]---*--- + if view > finalView { + // if epoch fallback is triggered, we treat this view as part of the last committed epoch + if lookup.epochFallbackIsTriggered.Load() { + return lookup.epochs.latest().counter, nil } - if err != nil { - return 0, fmt.Errorf("unexpected error in EECC logic while retrieving DKG data: %w", err) + // otherwise, we are waiting for the epoch including this view to be committed + return 0, model.ErrViewForUnknownEpoch + } + + // view is within a known epoch + for _, epoch := range lookup.epochs { + if !epoch.exists() { + continue + } + if epoch.firstView <= view && view <= epoch.finalView { + return epoch.counter, nil } } - // HAPPY PATH logic - return l.EpochForView(view) + // reaching this point indicates a corrupted state or internal bug + return 0, fmt.Errorf("sanity check failed: cached epochs (%v) does not contain input view %d", lookup.epochs, view) +} + +// handleProtocolEvents processes queued Epoch events `EpochCommittedPhaseStarted` +// and `EpochEmergencyFallbackTriggered`. This function permanently utilizes a worker +// routine until the `Component` terminates. +// When we observe a new epoch being committed, we compute +// the leader selection and cache static info for the epoch. When we observe +// epoch emergency fallback being triggered, we inject a fallback epoch. +func (lookup *EpochLookup) handleProtocolEvents(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + for { + select { + case <-ctx.Done(): + return + case block := <-lookup.committedEpochsCh: + epoch := lookup.state.AtBlockID(block.ID()).Epochs().Next() + err := lookup.cacheEpoch(epoch) + if err != nil { + ctx.Throw(err) + } + } + } +} + +// EpochCommittedPhaseStarted informs the `committee.Consensus` that the block starting the Epoch Committed Phase has been finalized. +func (lookup *EpochLookup) EpochCommittedPhaseStarted(_ uint64, first *flow.Header) { + lookup.committedEpochsCh <- first +} + +// EpochEmergencyFallbackTriggered passes the protocol event to the worker thread. +func (lookup *EpochLookup) EpochEmergencyFallbackTriggered() { + lookup.epochFallbackIsTriggered.Store(true) } diff --git a/module/epochs/epoch_lookup_test.go b/module/epochs/epoch_lookup_test.go index c16eb6a3ad0..7c278542e43 100644 --- a/module/epochs/epoch_lookup_test.go +++ b/module/epochs/epoch_lookup_test.go @@ -1,68 +1,310 @@ package epochs import ( + "context" + "sync" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/utils/unittest" "github.com/onflow/flow-go/utils/unittest/mocks" ) -func TestEpochForView(t *testing.T) { - // epoch 100: 0 -> 999 - // epoch 101: 1000 -> 1999 - // epoch 102: 2000 -> 2999 - epochs := []*mockprotocol.Epoch{} - for i := 0; i < 3; i++ { - firstView := i * 1000 - lastView := i*1000 + 999 - counter := 100 + i - epoch := new(mockprotocol.Epoch) - epoch.On("FirstView").Return(uint64(firstView), nil) - epoch.On("FinalView").Return(uint64(lastView), nil) - epoch.On("Counter").Return(uint64(counter), nil) - // return nil error to indicate a committed epoch - epoch.On("DKG").Return(nil, nil) - epochs = append(epochs, epoch) - } +type EpochLookupSuite struct { + suite.Suite - epochQuery := mocks.NewEpochQuery(t, 101) - for _, e := range epochs { - epochQuery.Add(e) - } + // mocks + epochQuery *mocks.EpochQuery + state *mockprotocol.State + snapshot *mockprotocol.Snapshot + params *mockprotocol.Params + + // backend for mocked functions + mu sync.Mutex // protects access to epochFallbackTriggered and phase + epochFallbackTriggered bool + phase flow.EpochPhase + + // config for each epoch + currentEpochCounter uint64 + prevEpoch epochRange + currEpoch epochRange + nextEpoch epochRange + + lookup *EpochLookup + cancel context.CancelFunc +} + +func TestEpochLookup(t *testing.T) { + suite.Run(t, new(EpochLookupSuite)) +} + +func (suite *EpochLookupSuite) SetupTest() { + suite.currentEpochCounter = uint64(1) + suite.phase = flow.EpochPhaseStaking - snapshot := new(mockprotocol.Snapshot) - snapshot.On("Epochs").Return(epochQuery) + suite.prevEpoch = epochRange{counter: suite.currentEpochCounter - 1, firstView: 100, finalView: 199} + suite.currEpoch = epochRange{counter: suite.currentEpochCounter, firstView: 200, finalView: 299} + suite.nextEpoch = epochRange{counter: suite.currentEpochCounter + 1, firstView: 300, finalView: 399} - state := new(mockprotocol.State) - state.On("Final").Return(snapshot) + suite.state = new(mockprotocol.State) + suite.snapshot = new(mockprotocol.Snapshot) + suite.params = new(mockprotocol.Params) + suite.epochQuery = mocks.NewEpochQuery(suite.T(), suite.currentEpochCounter) - lookup := NewEpochLookup(state) + suite.snapshot.On("Epochs").Return(suite.epochQuery) + suite.snapshot.On("Phase").Return( + func() flow.EpochPhase { return suite.Phase() }, + func() error { return nil }) - type testCase struct { - view uint64 - epoch uint64 + suite.params.On("EpochFallbackTriggered").Return( + func() bool { return suite.EpochFallbackTriggered() }, + func() error { return nil }) + + suite.state.On("Final").Return(suite.snapshot) + suite.state.On("Params").Return(suite.params) +} + +func (suite *EpochLookupSuite) TearDownTest() { + if suite.cancel != nil { + suite.cancel() } +} - testCases := []testCase{ - {view: 0, epoch: 100}, - {view: 500, epoch: 100}, - {view: 999, epoch: 100}, - {view: 1000, epoch: 101}, - {view: 1500, epoch: 101}, - {view: 1999, epoch: 101}, - {view: 2000, epoch: 102}, - {view: 2500, epoch: 102}, - {view: 2999, epoch: 102}, +// WithLock runs the given function while holding the suite lock. Must be used +// while updating fields used as backends for mocked functions. +func (suite *EpochLookupSuite) WithLock(f func()) { + suite.mu.Lock() + f() + suite.mu.Unlock() +} + +func (suite *EpochLookupSuite) EpochFallbackTriggered() bool { + suite.mu.Lock() + defer suite.mu.Unlock() + return suite.epochFallbackTriggered +} + +func (suite *EpochLookupSuite) Phase() flow.EpochPhase { + suite.mu.Lock() + defer suite.mu.Unlock() + return suite.phase +} + +// CommitEpochs adds the new epochs to the state. +func (suite *EpochLookupSuite) CommitEpochs(epochs ...epochRange) { + for _, epoch := range epochs { + mockEpoch := newMockEpoch(epoch.counter, epoch.firstView, epoch.finalView) + suite.epochQuery.Add(mockEpoch) + // if we add a next epoch (counter 1 greater than current), then set phase to committed + if epoch.counter == suite.currentEpochCounter+1 { + suite.WithLock(func() { + suite.phase = flow.EpochPhaseCommitted + }) + } } +} + +// CreateAndStartEpochLookup instantiates and starts the lookup. +// Should be called only once per test, after initial epoch mocks are created. +// It spawns a goroutine to detect fatal errors from the committee's error channel. +func (suite *EpochLookupSuite) CreateAndStartEpochLookup() { + lookup, err := NewEpochLookup(suite.state) + suite.Require().NoError(err) + ctx, cancel, errCh := irrecoverable.WithSignallerAndCancel(context.Background()) + lookup.Start(ctx) + go unittest.FailOnIrrecoverableError(suite.T(), ctx.Done(), errCh) + + suite.lookup = lookup + suite.cancel = cancel +} + +// TestEpochForViewWithFallback_Curr tests constructing and subsequently querying +// EpochLookup with an initial state of a current epoch. +func (suite *EpochLookupSuite) TestEpochForViewWithFallback_Curr() { + epochs := []epochRange{suite.currEpoch} + suite.CommitEpochs(epochs...) + suite.CreateAndStartEpochLookup() + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, epochs...) +} + +// TestEpochForViewWithFallback_PrevCurr tests constructing and subsequently querying +// EpochLookup with an initial state of a previous and current epoch. +func (suite *EpochLookupSuite) TestEpochForViewWithFallback_PrevCurr() { + epochs := []epochRange{suite.prevEpoch, suite.currEpoch} + suite.CommitEpochs(epochs...) + suite.CreateAndStartEpochLookup() + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, epochs...) +} - for _, tc := range testCases { - epoch, err := lookup.EpochForView(tc.view) - require.NoError(t, err) - require.Equal(t, tc.epoch, epoch) - epoch, err = lookup.EpochForViewWithFallback(tc.view) - require.NoError(t, err) - require.Equal(t, tc.epoch, epoch) +// TestEpochForViewWithFallback_CurrNext tests constructing and subsequently querying +// EpochLookup with an initial state of a current and next epoch. +func (suite *EpochLookupSuite) TestEpochForViewWithFallback_CurrNext() { + epochs := []epochRange{suite.currEpoch, suite.nextEpoch} + suite.CommitEpochs(epochs...) + suite.CreateAndStartEpochLookup() + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, epochs...) +} + +// TestEpochForViewWithFallback_CurrNextPrev tests constructing and subsequently querying +// EpochLookup with an initial state of a previous, current, and next epoch. +func (suite *EpochLookupSuite) TestEpochForViewWithFallback_CurrNextPrev() { + epochs := []epochRange{suite.prevEpoch, suite.currEpoch, suite.nextEpoch} + suite.CommitEpochs(epochs...) + suite.CreateAndStartEpochLookup() + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, epochs...) +} + +// TestEpochForViewWithFallback_EpochFallbackTriggered tests constructing and subsequently querying +// EpochLookup with an initial state of epoch fallback triggered. +func (suite *EpochLookupSuite) TestEpochForViewWithFallback_EpochFallbackTriggered() { + epochs := []epochRange{suite.prevEpoch, suite.currEpoch, suite.nextEpoch} + suite.WithLock(func() { + suite.epochFallbackTriggered = true + }) + suite.CommitEpochs(epochs...) + suite.CreateAndStartEpochLookup() + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, epochs...) +} + +// TestProtocolEvents_EpochFallbackTriggered tests constructing and subsequently querying +// EpochLookup, where there is no epoch fallback at construction time, +// but an epoch fallback happens later via an epoch event. +func (suite *EpochLookupSuite) TestProtocolEvents_EpochFallbackTriggered() { + // initially, only current epoch is committed + suite.CommitEpochs(suite.currEpoch) + suite.CreateAndStartEpochLookup() + + // trigger epoch fallback + suite.WithLock(func() { + suite.epochFallbackTriggered = true + }) + suite.lookup.EpochEmergencyFallbackTriggered() + + // wait for the protocol event to be processed (async) + assert.Eventually(suite.T(), func() bool { + _, err := suite.lookup.EpochForViewWithFallback(suite.currEpoch.finalView + 1) + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + // validate queries are answered correctly + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, suite.currEpoch) + + // should handle multiple deliveries of the protocol event + suite.lookup.EpochEmergencyFallbackTriggered() + suite.lookup.EpochEmergencyFallbackTriggered() + suite.lookup.EpochEmergencyFallbackTriggered() + + // validate queries are answered correctly + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, suite.currEpoch) +} + +// TestProtocolEvents_CommittedEpoch tests correct processing of an `EpochCommittedPhaseStarted` event +func (suite *EpochLookupSuite) TestProtocolEvents_CommittedEpoch() { + // initially, only current epoch is committed + suite.CommitEpochs(suite.currEpoch) + suite.CreateAndStartEpochLookup() + + // commit the next epoch, and emit a protocol event + firstBlockOfCommittedPhase := unittest.BlockHeaderFixture() + suite.state.On("AtBlockID", firstBlockOfCommittedPhase.ID()).Return(suite.snapshot) + suite.CommitEpochs(suite.nextEpoch) + suite.lookup.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + + // wait for the protocol event to be processed (async) + assert.Eventually(suite.T(), func() bool { + _, err := suite.lookup.EpochForViewWithFallback(suite.currEpoch.finalView + 1) + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + // validate queries are answered correctly + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, suite.currEpoch, suite.nextEpoch) + + // should handle multiple deliveries of the protocol event + suite.lookup.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + suite.lookup.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + suite.lookup.EpochCommittedPhaseStarted(suite.currentEpochCounter, firstBlockOfCommittedPhase) + + // validate queries are answered correctly + testEpochForViewWithFallback(suite.T(), suite.lookup, suite.state, suite.currEpoch, suite.nextEpoch) +} + +// testEpochForViewWithFallback accepts a constructed EpochLookup and state, and +// validates correctness by issuing various queries, using the input state and +// epochs as source of truth. +func testEpochForViewWithFallback(t *testing.T, lookup *EpochLookup, state protocol.State, epochs ...epochRange) { + epochFallbackTriggered, err := state.Params().EpochFallbackTriggered() + require.NoError(t, err) + + t.Run("should have set epoch fallback triggered correctly", func(t *testing.T) { + assert.Equal(t, epochFallbackTriggered, lookup.epochFallbackIsTriggered.Load()) + }) + + t.Run("should be able to query within any committed epoch", func(t *testing.T) { + for _, epoch := range epochs { + t.Run("first view", func(t *testing.T) { + counter, err := lookup.EpochForViewWithFallback(epoch.firstView) + assert.NoError(t, err) + assert.Equal(t, epoch.counter, counter) + }) + t.Run("final view", func(t *testing.T) { + counter, err := lookup.EpochForViewWithFallback(epoch.finalView) + assert.NoError(t, err) + assert.Equal(t, epoch.counter, counter) + }) + t.Run("random view in range", func(t *testing.T) { + counter, err := lookup.EpochForViewWithFallback(unittest.Uint64InRange(epoch.firstView, epoch.finalView)) + assert.NoError(t, err) + assert.Equal(t, epoch.counter, counter) + }) + } + }) + + t.Run("should return ErrViewForUnknownEpoch below earliest epoch", func(t *testing.T) { + t.Run("view 0", func(t *testing.T) { + _, err := lookup.EpochForViewWithFallback(0) + assert.ErrorIs(t, err, model.ErrViewForUnknownEpoch) + }) + t.Run("boundary of earliest epoch", func(t *testing.T) { + _, err := lookup.EpochForViewWithFallback(epochs[0].firstView - 1) + assert.ErrorIs(t, err, model.ErrViewForUnknownEpoch) + }) + t.Run("random view below earliest epoch", func(t *testing.T) { + _, err := lookup.EpochForViewWithFallback(unittest.Uint64InRange(0, epochs[0].firstView-1)) + assert.ErrorIs(t, err, model.ErrViewForUnknownEpoch) + }) + }) + + // if epoch fallback is triggered, fallback to returning latest epoch counter + // otherwise return ErrViewForUnknownEpoch + if epochFallbackTriggered { + t.Run("should use fallback logic for queries above latest epoch when epoch fallback is triggered", func(t *testing.T) { + counter, err := lookup.EpochForViewWithFallback(epochs[len(epochs)-1].finalView + 1) + assert.NoError(t, err) + // should fallback to returning the counter for the latest epoch + assert.Equal(t, epochs[len(epochs)-1].counter, counter) + }) + } else { + t.Run("should return ErrViewForUnknownEpoch for queries above latest epoch when epoch fallback is not triggered", func(t *testing.T) { + _, err := lookup.EpochForViewWithFallback(epochs[len(epochs)-1].finalView + 1) + assert.ErrorIs(t, err, model.ErrViewForUnknownEpoch) + }) } } + +// newMockEpoch returns a mock epoch with the given fields set. +func newMockEpoch(counter, firstView, finalView uint64) *mockprotocol.Epoch { + epoch := new(mockprotocol.Epoch) + epoch.On("FirstView").Return(firstView, nil) + epoch.On("FinalView").Return(finalView, nil) + epoch.On("Counter").Return(counter, nil) + return epoch +} diff --git a/module/epochs/errors.go b/module/epochs/errors.go new file mode 100644 index 00000000000..34035635ed6 --- /dev/null +++ b/module/epochs/errors.go @@ -0,0 +1,33 @@ +package epochs + +import ( + "errors" + "fmt" +) + +// ClusterQCNoVoteError is returned when a ClusterRootQCVoter fails to submit a vote +// for reasons we consider benign, either: +// - we have reached a retry limit for transient failures +// - we have decided to not submit the vote, for example because we are not in the EpochSetup phase +type ClusterQCNoVoteError struct { + Err error +} + +func (err ClusterQCNoVoteError) Error() string { + return err.Err.Error() +} + +func (err ClusterQCNoVoteError) Unwrap() error { + return err.Err +} + +func NewClusterQCNoVoteErrorf(msg string, args ...interface{}) ClusterQCNoVoteError { + return ClusterQCNoVoteError{ + Err: fmt.Errorf(msg, args...), + } +} + +func IsClusterQCNoVoteError(err error) bool { + var errClusterVotingFailed ClusterQCNoVoteError + return errors.As(err, &errClusterVotingFailed) +} diff --git a/module/epochs/qc_client.go b/module/epochs/qc_client.go index 3617a481ebc..a1a2b5ec461 100644 --- a/module/epochs/qc_client.go +++ b/module/epochs/qc_client.go @@ -3,6 +3,7 @@ package epochs import ( "context" "encoding/hex" + "errors" "fmt" "time" @@ -12,6 +13,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/consensus/hotstuff/model" hotstuffver "github.com/onflow/flow-go/consensus/hotstuff/verification" @@ -70,6 +72,11 @@ func NewQCContractClient( // contract. This function returns only once the transaction has been // processed by the network. An error is returned if the transaction has // failed and should be re-submitted. +// Error returns: +// - network.TransientError for any errors from the underlying client, if the retry period has been exceeded +// - errTransactionExpired if the transaction has expired +// - errTransactionReverted if the transaction execution reverted +// - generic error in case of unexpected critical failure func (c *QCContractClient) SubmitVote(ctx context.Context, vote *model.Vote) error { // time method was invoked @@ -82,13 +89,15 @@ func (c *QCContractClient) SubmitVote(ctx context.Context, vote *model.Vote) err // get account for given address and also validates AccountKeyIndex is valid account, err := c.GetAccount(ctx) if err != nil { - return fmt.Errorf("could not get account: %w", err) + // we consider all errors from client network calls to be transient and non-critical + return network.NewTransientErrorf("could not get account: %w", err) } // get latest finalized block to execute transaction latestBlock, err := c.FlowClient.GetLatestBlock(ctx, false) if err != nil { - return fmt.Errorf("could not get latest block from node: %w", err) + // we consider all errors from client network calls to be transient and non-critical + return network.NewTransientErrorf("could not get latest block from node: %w", err) } // attach submit vote transaction template and build transaction @@ -132,11 +141,19 @@ func (c *QCContractClient) SubmitVote(ctx context.Context, vote *model.Vote) err c.Log.Info().Str("tx_id", tx.ID().Hex()).Msg("sending SubmitResult transaction") txID, err := c.SendTransaction(ctx, tx) if err != nil { + // context expiring is not a critical failure, wrap as transient + if errors.Is(err, ctx.Err()) { + return network.NewTransientErrorf("failed to submit transaction: context done: %w", err) + } return fmt.Errorf("failed to submit transaction: %w", err) } err = c.WaitForSealed(ctx, txID, started) if err != nil { + // context expiring is not a critical failure, wrap as transient + if errors.Is(err, ctx.Err()) { + return network.NewTransientErrorf("failed to submit transaction: context done: %w", err) + } return fmt.Errorf("failed to wait for transaction seal: %w", err) } @@ -145,19 +162,27 @@ func (c *QCContractClient) SubmitVote(ctx context.Context, vote *model.Vote) err // Voted returns true if we have successfully submitted a vote to the // cluster QC aggregator smart contract for the current epoch. +// Error returns: +// - network.TransientError for any errors from the underlying Flow client +// - generic error in case of unexpected critical failures func (c *QCContractClient) Voted(ctx context.Context) (bool, error) { // execute script to read if voted template := templates.GenerateGetNodeHasVotedScript(c.env) - hasVoted, err := c.FlowClient.ExecuteScriptAtLatestBlock(ctx, template, []cadence.Value{cadence.String(c.nodeID.String())}) + ret, err := c.FlowClient.ExecuteScriptAtLatestBlock(ctx, template, []cadence.Value{cadence.String(c.nodeID.String())}) if err != nil { - return false, fmt.Errorf("could not execute voted script: %w", err) + // we consider all errors from client network calls to be transient and non-critical + return false, network.NewTransientErrorf("could not execute voted script: %w", err) + } + + voted, ok := ret.(cadence.Bool) + if !ok { + return false, fmt.Errorf("unexpected cadence type (%T) returned from Voted script", ret) } // check if node has voted - if !hasVoted.(cadence.Bool) { + if !voted { return false, nil } - return true, nil } diff --git a/module/epochs/qc_voter.go b/module/epochs/qc_voter.go index eb0d30f2340..9213e2165d1 100644 --- a/module/epochs/qc_voter.go +++ b/module/epochs/qc_voter.go @@ -2,11 +2,13 @@ package epochs import ( "context" + "errors" "fmt" "sync" "time" "github.com/onflow/flow-go/module/retrymiddleware" + "github.com/onflow/flow-go/network" "github.com/sethvargo/go-retry" @@ -70,11 +72,12 @@ func NewRootQCVoter( } // Vote handles the full procedure of generating a vote, submitting it to the -// epoch smart contract, and verifying submission. Returns an error only if -// there is a critical error that would make it impossible for the vote to be -// submitted. Otherwise, exits when the vote has been successfully submitted. -// +// epoch smart contract, and verifying submission. // It is safe to run multiple times within a single setup phase. +// +// Error returns: +// - ErrWontVote if we fail to vote for a benign reason +// - generic error in case of critical unexpected failure func (voter *RootQCVoter) Vote(ctx context.Context, epoch protocol.Epoch) error { counter, err := epoch.Counter() @@ -87,7 +90,7 @@ func (voter *RootQCVoter) Vote(ctx context.Context, epoch protocol.Epoch) error } cluster, clusterIndex, ok := clusters.ByNodeID(voter.me.NodeID()) if !ok { - return fmt.Errorf("could not find self in clustering") + return NewClusterQCNoVoteErrorf("could not find self in clustering") } log := voter.log.With(). @@ -125,16 +128,19 @@ func (voter *RootQCVoter) Vote(ctx context.Context, epoch protocol.Epoch) error // submit a vote anyway and must exit this process phase, err := voter.state.Final().Phase() if err != nil { - log.Error().Err(err).Msg("could not get current phase") + return fmt.Errorf("unexpected error - unable to get current epoch phase: %w", err) } else if phase != flow.EpochPhaseSetup { - return fmt.Errorf("could not submit vote - no longer in setup phase") + return NewClusterQCNoVoteErrorf("could not submit vote because we we are not in EpochSetup phase (in %s phase instead)", phase) } // check whether we've already voted, if we have we can exit early voted, err := qcContractClient.Voted(ctx) if err != nil { - log.Error().Err(err).Msg("could not check vote status") - return retry.RetryableError(err) + if network.IsTransientError(err) { + log.Warn().Err(err).Msg("unable to check vote status, retrying...") + return retry.RetryableError(err) + } + return fmt.Errorf("unexpected error in Voted script execution: %w", err) } else if voted { log.Info().Msg("already voted - exiting QC vote process...") // update our last successful client index for future calls @@ -147,8 +153,16 @@ func (voter *RootQCVoter) Vote(ctx context.Context, epoch protocol.Epoch) error log.Info().Msg("submitting vote...") err = qcContractClient.SubmitVote(ctx, vote) if err != nil { - log.Error().Err(err).Msg("could not submit vote - retrying...") - return retry.RetryableError(err) + if network.IsTransientError(err) || errors.Is(err, errTransactionExpired) { + log.Warn().Err(err).Msg("could not submit vote due to transient failure - retrying...") + return retry.RetryableError(err) + } else if errors.Is(err, errTransactionReverted) { + // this error case could be benign or not - if we observe it, we should investigate further + log.Err(err).Msg("vote submission failed due to execution failure - caution: this could be either a benign error (eg. 'already voted') or a critical bug - retrying") + return retry.RetryableError(err) + } else { + return fmt.Errorf("unexpected error submitting vote: %w", err) + } } log.Info().Msg("successfully submitted vote - exiting QC vote process...") @@ -157,7 +171,9 @@ func (voter *RootQCVoter) Vote(ctx context.Context, epoch protocol.Epoch) error voter.updateLastSuccessfulClient(clientIndex) return nil }) - + if network.IsTransientError(err) || errors.Is(err, errTransactionReverted) || errors.Is(err, errTransactionReverted) { + return NewClusterQCNoVoteErrorf("exceeded retry limit without successfully submitting our vote: %w", err) + } return err } diff --git a/module/epochs/qc_voter_test.go b/module/epochs/qc_voter_test.go index 414a01e7d2b..71a2fdd3b97 100644 --- a/module/epochs/qc_voter_test.go +++ b/module/epochs/qc_voter_test.go @@ -2,7 +2,6 @@ package epochs_test import ( "context" - "fmt" "io" "math/rand" "testing" @@ -78,7 +77,7 @@ func (suite *Suite) SetupTest() { var err error assignments := unittest.ClusterAssignment(2, suite.nodes) suite.clustering, err = factory.NewClusterList(assignments, suite.nodes) - suite.Require().Nil(err) + suite.Require().NoError(err) suite.epoch.On("Counter").Return(suite.counter, nil) suite.epoch.On("Clustering").Return(suite.clustering, nil) @@ -97,8 +96,8 @@ func (suite *Suite) TestNonClusterParticipant() { // change our identity so we aren't in the cluster assignment suite.me = unittest.IdentityFixture(unittest.WithRole(flow.RoleCollection)) err := suite.voter.Vote(context.Background(), suite.epoch) - fmt.Println(err) suite.Assert().Error(err) + suite.Assert().True(epochs.IsClusterQCNoVoteError(err)) } // should fail if we are not in setup phase @@ -106,8 +105,8 @@ func (suite *Suite) TestInvalidPhase() { suite.phase = flow.EpochPhaseStaking err := suite.voter.Vote(context.Background(), suite.epoch) - fmt.Println(err) suite.Assert().Error(err) + suite.Assert().True(epochs.IsClusterQCNoVoteError(err)) } // should succeed and exit if we've already voted @@ -115,12 +114,11 @@ func (suite *Suite) TestAlreadyVoted() { suite.voted = true err := suite.voter.Vote(context.Background(), suite.epoch) - fmt.Println(err) - suite.Assert().Nil(err) + suite.Assert().NoError(err) } // should succeed and exit if voting succeeds func (suite *Suite) TestVoting() { err := suite.voter.Vote(context.Background(), suite.epoch) - suite.Assert().Nil(err) + suite.Assert().NoError(err) } diff --git a/module/executiondatasync/execution_data/util.go b/module/executiondatasync/execution_data/util.go new file mode 100644 index 00000000000..f2585f4c61f --- /dev/null +++ b/module/executiondatasync/execution_data/util.go @@ -0,0 +1,19 @@ +package execution_data + +import ( + "context" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/blobs" +) + +func CalculateID(ctx context.Context, execData *BlockExecutionData, serializer Serializer) (flow.Identifier, error) { + executionDatastore := NewExecutionDataStore(&blobs.NoopBlobstore{}, serializer) + + id, err := executionDatastore.AddExecutionData(ctx, execData) + if err != nil { + return flow.ZeroID, err + } + + return id, nil +} diff --git a/module/finalizer.go b/module/finalizer.go index f837ce40302..274ef9b853a 100644 --- a/module/finalizer.go +++ b/module/finalizer.go @@ -7,24 +7,19 @@ import ( ) // Finalizer is used by the consensus algorithm to inform other components for (such -// as the protocol state) about validity of block headers and finalization of blocks. +// as the protocol state) about finalization of blocks. // // Since we have two different protocol states: one for the main consensus, // the other for the collection cluster consensus, the Finalizer interface // allows the two different protocol states to provide different implementations -// for updating its state when a block has been validated or finalized. +// for updating its state when a block has been finalized. // -// Why MakeValid and MakeFinal need to return an error? // Updating the protocol state should always succeed when the data is consistent. // However, in case the protocol state is corrupted, error should be returned and -// the consensus algorithm should halt. So the error returned from MakeValid and -// MakeFinal is for the protocol state to report exceptions. +// the consensus algorithm should halt. So the error returned from MakeFinal is +// for the protocol state to report exceptions. type Finalizer interface { - // MakeValid will mark a block as having passed the consensus algorithm's - // internal validation. - MakeValid(blockID flow.Identifier) error - // MakeFinal will declare a block and all of its ancestors as finalized, which // makes it an immutable part of the blockchain. Returning an error indicates // some fatal condition and will cause the finalization logic to terminate. diff --git a/module/finalizer/collection/finalizer.go b/module/finalizer/collection/finalizer.go index 71b359f0a94..bfe1d76ae4f 100644 --- a/module/finalizer/collection/finalizer.go +++ b/module/finalizer/collection/finalizer.go @@ -52,6 +52,7 @@ func NewFinalizer( // included in a block proposal. Between entering the non-finalized chain state // and being finalized, entities should be present in both the volatile memory // pools and persistent storage. +// No errors are expected during normal operation. func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { return operation.RetryOnConflict(f.db.Update, func(tx *badger.Txn) error { @@ -173,13 +174,3 @@ func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { return nil }) } - -// MakeValid marks a block as having passed HotStuff validation. -func (f *Finalizer) MakeValid(blockID flow.Identifier) error { - return operation.RetryOnConflict( - f.db.Update, - operation.SkipDuplicates( - operation.InsertBlockValidity(blockID, true), - ), - ) -} diff --git a/module/finalizer/collection/finalizer_test.go b/module/finalizer/collection/finalizer_test.go index 36768503bd5..921e8cc6c57 100644 --- a/module/finalizer/collection/finalizer_test.go +++ b/module/finalizer/collection/finalizer_test.go @@ -53,7 +53,7 @@ func TestFinalizer(t *testing.T) { // a helper function to bootstrap with the genesis block bootstrap := func() { - stateRoot, err := cluster.NewStateRoot(genesis) + stateRoot, err := cluster.NewStateRoot(genesis, unittest.QuorumCertificateFixture()) require.NoError(t, err) state, err = cluster.Bootstrap(db, stateRoot) require.NoError(t, err) diff --git a/module/finalizer/consensus/finalizer.go b/module/finalizer/consensus/finalizer.go index 8fa91e3a6c2..5d68a97cf90 100644 --- a/module/finalizer/consensus/finalizer.go +++ b/module/finalizer/consensus/finalizer.go @@ -52,6 +52,7 @@ func NewFinalizer(db *badger.DB, // included in a block proposal. Between entering the non-finalized chain state // and being finalized, entities should be present in both the volatile memory // pools and persistent storage. +// No errors are expected during normal operation. func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { span, ctx := f.tracer.StartBlockSpan(context.Background(), blockID, trace.CONFinalizerFinalizeBlock) @@ -126,12 +127,3 @@ func (f *Finalizer) MakeFinal(blockID flow.Identifier) error { return nil } - -// MakeValid marks a block as having passed HotStuff validation. -func (f *Finalizer) MakeValid(blockID flow.Identifier) error { - err := f.state.MarkValid(blockID) - if err != nil { - return fmt.Errorf("could not mark block as valid (%x): %w", blockID, err) - } - return nil -} diff --git a/module/forest/leveled_forrest.go b/module/forest/leveled_forest.go similarity index 52% rename from module/forest/leveled_forrest.go rename to module/forest/leveled_forest.go index fc5a615a3e4..6a008c76ca6 100644 --- a/module/forest/leveled_forrest.go +++ b/module/forest/leveled_forest.go @@ -1,18 +1,26 @@ package forest import ( - "fmt" - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/mempool" ) // LevelledForest contains multiple trees (which is a potentially disconnected planar graph). -// Each vertexContainer in the graph has a level (view) and a hash. A vertexContainer can only have one parent -// with strictly smaller level (view). A vertexContainer can have multiple children, all with -// strictly larger level (view). +// Each vertex in the graph has a level and a hash. A vertex can only have one parent, which +// must have strictly smaller level. A vertex can have multiple children, all with strictly +// larger level. // A LevelledForest provides the ability to prune all vertices up to a specific level. // A tree whose root is below the pruning threshold might decompose into multiple // disconnected subtrees as a result of pruning. +// By design, the LevelledForest does _not_ touch the parent information for vertices +// that are on the lowest retained level. Thereby, it is possible to initialize the +// LevelledForest with a root vertex at the lowest retained level, without this root +// needing to have a parent. Furthermore, the root vertex can be at level 0 and in +// absence of a parent still satisfy the condition that any parent must be of lower level +// (mathematical principle of vacuous truth) without the implementation needing to worry +// about unsigned integer underflow. +// +// LevelledForest is NOT safe for concurrent use by multiple goroutines. type LevelledForest struct { vertices VertexSet verticesAtLevel map[uint64]VertexList @@ -48,10 +56,12 @@ func NewLevelledForest(lowestLevel uint64) *LevelledForest { } } -// PruneUpToLevel prunes all blocks UP TO but NOT INCLUDING `level` +// PruneUpToLevel prunes all blocks UP TO but NOT INCLUDING `level`. +// Error returns: +// * mempool.BelowPrunedThresholdError if input level is below the lowest retained level func (f *LevelledForest) PruneUpToLevel(level uint64) error { if level < f.LowestLevel { - return fmt.Errorf("new lowest level %d cannot be smaller than previous last retained level %d", level, f.LowestLevel) + return mempool.NewBelowPrunedThresholdErrorf("new lowest level %d cannot be smaller than previous last retained level %d", level, f.LowestLevel) } if len(f.vertices) == 0 { f.LowestLevel = level @@ -94,7 +104,7 @@ func (f *LevelledForest) PruneUpToLevel(level uint64) error { return nil } -// HasVertex returns true iff full vertex exists +// HasVertex returns true iff full vertex exists. func (f *LevelledForest) HasVertex(id flow.Identifier) bool { container, exists := f.vertices[id] return exists && !f.isEmptyContainer(container) @@ -115,23 +125,23 @@ func (f *LevelledForest) GetVertex(id flow.Identifier) (Vertex, bool) { return container.vertex, true } -// GetSize returns the total number of vertices above the pruned lowest level. +// GetSize returns the total number of vertices above the lowest pruned level. // Note this call is not concurrent-safe, caller is responsible to ensure concurrency safety. func (f *LevelledForest) GetSize() uint64 { return f.size } // GetChildren returns a VertexIterator to iterate over the children -// An empty VertexIterator is returned, if no vertices are known whose parent is `id` , `level` +// An empty VertexIterator is returned, if no vertices are known whose parent is `id`. func (f *LevelledForest) GetChildren(id flow.Identifier) VertexIterator { container := f.vertices[id] - // if vertex does not exists, container is the default zero value for vertexContainer, which contains a nil-slice for its children + // if vertex does not exist, container is the default zero value for vertexContainer, which contains a nil-slice for its children return newVertexIterator(container.children) // VertexIterator gracefully handles nil slices } // GetNumberOfChildren returns number of children of given vertex func (f *LevelledForest) GetNumberOfChildren(id flow.Identifier) int { - container := f.vertices[id] // if vertex does not exists, container is the default zero value for vertexContainer, which contains a nil-slice for its children + container := f.vertices[id] // if vertex does not exist, container is the default zero value for vertexContainer, which contains a nil-slice for its children num := 0 for _, child := range container.children { if child.vertex != nil { @@ -141,17 +151,24 @@ func (f *LevelledForest) GetNumberOfChildren(id flow.Identifier) int { return num } -// GetVerticesAtLevel returns a VertexIterator to iterate over the Vertices at the specified height -// An empty VertexIterator is returned, if no vertices are known at the specified `level` +// GetVerticesAtLevel returns a VertexIterator to iterate over the Vertices at the specified level. +// An empty VertexIterator is returned, if no vertices are known at the specified level. +// If `level` is already pruned, an empty VertexIterator is returned. func (f *LevelledForest) GetVerticesAtLevel(level uint64) VertexIterator { return newVertexIterator(f.verticesAtLevel[level]) // go returns the zero value for a missing level. Here, a nil slice } -// GetNumberOfVerticesAtLevel returns number of vertices at given level +// GetNumberOfVerticesAtLevel returns the number of full vertices at given level. +// A full vertex is a vertex that was explicitly added to the forest. In contrast, +// an empty vertex container represents a vertex that is _referenced_ as parent by +// one or more full vertices, but has not been added itself to the forest. +// We only count vertices that have been explicitly added to the forest and not yet +// pruned. (In comparision, we do _not_ count vertices that are _referenced_ as +// parent by vertices, but have not been added themselves). func (f *LevelledForest) GetNumberOfVerticesAtLevel(level uint64) int { num := 0 for _, container := range f.verticesAtLevel[level] { - if container.vertex != nil { + if !f.isEmptyContainer(container) { num++ } } @@ -196,7 +213,6 @@ func (f *LevelledForest) registerWithParent(vertexContainer *vertexContainer) { // getOrCreateVertexContainer returns the vertexContainer if there exists one // or creates a new vertexContainer and adds it to the internal data structures. -// It errors if a vertex with same id but different Level is already known // (i.e. there exists an empty or full container with the same id but different level). func (f *LevelledForest) getOrCreateVertexContainer(id flow.Identifier, level uint64) *vertexContainer { container, exists := f.vertices[id] // try to find vertex container with same ID @@ -206,92 +222,104 @@ func (f *LevelledForest) getOrCreateVertexContainer(id flow.Identifier, level ui level: level, } f.vertices[container.id] = container - vtcs := f.verticesAtLevel[container.level] // returns nil slice if not yet present - f.verticesAtLevel[container.level] = append(vtcs, container) // append works on nil slices: creates slice with capacity 2 + vertices := f.verticesAtLevel[container.level] // returns nil slice if not yet present + f.verticesAtLevel[container.level] = append(vertices, container) // append works on nil slices: creates slice with capacity 2 } return container } -// VerifyVertex verifies that vertex satisfies the following conditions -// (1) -func (f *LevelledForest) VerifyVertex(vertex Vertex) error { - if vertex.Level() < f.LowestLevel { - return nil - } - isKnownVertex, err := f.isEquivalentToStoredVertex(vertex) - if err != nil { - return fmt.Errorf("invalid Vertex: %w", err) - } - if isKnownVertex { +// VerifyVertex verifies that adding vertex `v` would yield a valid Levelled Forest. +// Specifically, we verify that _all_ of the following conditions are satisfied: +// +// 1. `v.Level()` must be strictly larger than the level that `v` reports +// for its parent (maintains an acyclic graph). +// +// 2. If a vertex with the same ID as `v.VertexID()` exists in the graph or is +// referenced by another vertex within the graph, the level must be identical. +// (In other words, we don't have vertices with the same ID but different level) +// +// 3. Let `ParentLevel`, `ParentID` denote the level, ID that `v` reports for its parent. +// If a vertex with `ParentID` exists (or is referenced by other vertices as their parent), +// we require that the respective level is identical to `ParentLevel`. +// +// Notes: +// - If `v.Level()` has already been pruned, adding it to the forest is a NoOp. +// Hence, any vertex with level below the pruning threshold automatically passes. +// - By design, the LevelledForest does _not_ touch the parent information for vertices +// that are on the lowest retained level. Thereby, it is possible to initialize the +// LevelledForest with a root vertex at the lowest retained level, without this root +// needing to have a parent. Furthermore, the root vertex can be at level 0 and in +// absence of a parent still satisfy the condition that any parent must be of lower level +// (mathematical principle of vacuous truth) without the implementation needing to worry +// about unsigned integer underflow. +// +// Error returns: +// * InvalidVertexError if the input vertex is invalid for insertion to the forest. +func (f *LevelledForest) VerifyVertex(v Vertex) error { + if v.Level() < f.LowestLevel { return nil } - // vertex not found in storage => new vertex - // verify new vertex - if vertex.Level() == f.LowestLevel { - return nil + storedContainer, haveVertexContainer := f.vertices[v.VertexID()] + if !haveVertexContainer { // have no vertex with same id stored + // the only thing remaining to check is the parent information + return f.ensureConsistentParent(v) } - return f.verifyParent(vertex) -} -// isEquivalentToStoredVertex -// evaluates whether a vertex is equivalent to already stored vertex. -// for vertices at pruning level, parents are ignored -// -// (1) return value (false, nil) -// Two vertices are _not equivalent_ if they have different IDs (Hashes). -// -// (2) return value (true, nil) -// Two vertices _are equivalent_ if their respective fields are identical: -// ID, Level, and Parent (both parent ID and parent Level) -// -// (3) return value (false, error) -// errors if the vertices' IDs are identical but they differ -// in any of the _relevant_ fields (as defined in (2)). -func (f *LevelledForest) isEquivalentToStoredVertex(vertex Vertex) (bool, error) { - storedVertex, haveStoredVertex := f.GetVertex(vertex.VertexID()) - if !haveStoredVertex { - return false, nil //have no vertex with same id stored + // Found a vertex container, i.e. `v` already exists, or it is referenced by some other vertex. + // In all cases, `v.Level()` should match the vertexContainer's information + if v.Level() != storedContainer.level { + return NewInvalidVertexErrorf(v, "level conflicts with stored vertex with same id (%d!=%d)", v.Level(), storedContainer.level) } - // found vertex in storage with identical ID - // => we expect all other (relevant) fields to be identical - if vertex.Level() != storedVertex.Level() { // view number - return false, fmt.Errorf("conflicting vertices with ID %v", vertex.VertexID()) + // vertex container is empty, i.e. `v` is referenced by some other vertex as its parent: + if f.isEmptyContainer(storedContainer) { + // the only thing remaining to check is the parent information + return f.ensureConsistentParent(v) } - if vertex.Level() <= f.LowestLevel { - return true, nil + + // vertex container holds a vertex with the same ID as `v`: + // The parent information from vertexContainer has already been checked for consistency. So + // we simply compare with the existing vertex for inconsistencies + + // the vertex is at or below the lowest retained level, so we can't check the parent (it's pruned) + if v.Level() == f.LowestLevel { + return nil } - newParentId, newParentView := vertex.Parent() - storedParentId, storedParentView := storedVertex.Parent() - if newParentId != storedParentId { // qc.blockID - return false, fmt.Errorf("conflicting vertices with ID %v", vertex.VertexID()) + + newParentId, newParentLevel := v.Parent() + storedParentId, storedParentLevel := storedContainer.vertex.Parent() + if newParentId != storedParentId { + return NewInvalidVertexErrorf(v, "parent ID conflicts with stored parent (%x!=%x)", newParentId, storedParentId) } - if newParentView != storedParentView { // qc.view - return false, fmt.Errorf("conflicting vertices with ID %v", vertex.VertexID()) + if newParentLevel != storedParentLevel { + return NewInvalidVertexErrorf(v, "parent level conflicts with stored parent (%d!=%d)", newParentLevel, storedParentLevel) } // all _relevant_ fields identical - return true, nil + return nil } -// verifyParent verifies whether vertex.Parent() is consistent with current forest. -// An error is raised if -// * there is a parent with the same id but different view; +// ensureConsistentParent verifies that vertex.Parent() is consistent with current forest. +// Returns InvalidVertexError if: +// * there is a parent with the same ID but different level; // * the parent's level is _not_ smaller than the vertex's level -func (f *LevelledForest) verifyParent(vertex Vertex) error { +func (f *LevelledForest) ensureConsistentParent(vertex Vertex) error { + if vertex.Level() <= f.LowestLevel { + // the vertex is at or below the lowest retained level, so we can't check the parent (it's pruned) + return nil + } + // verify parent parentID, parentLevel := vertex.Parent() if !(vertex.Level() > parentLevel) { - return fmt.Errorf("parent vertex's level (%d) must be smaller than the vertex's level (%d)", parentLevel, vertex.Level()) + return NewInvalidVertexErrorf(vertex, "vertex parent level (%d) must be smaller than proposed vertex level (%d)", parentLevel, vertex.Level()) } - parentVertex, haveParentStored := f.GetVertex(parentID) + storedParent, haveParentStored := f.GetVertex(parentID) if !haveParentStored { return nil } - if parentVertex.Level() != parentLevel { - return fmt.Errorf("parent vertex of %v has different level (%d) than the stored vertex (%d)", - vertex.VertexID(), parentLevel, parentVertex.Level(), - ) + if storedParent.Level() != parentLevel { + return NewInvalidVertexErrorf(vertex, "parent level conflicts with stored parent (%d!=%d)", parentLevel, storedParent.Level()) } return nil } diff --git a/module/forest/leveled_forrest_test.go b/module/forest/leveled_forest_test.go similarity index 70% rename from module/forest/leveled_forrest_test.go rename to module/forest/leveled_forest_test.go index e4f26469075..c0285918167 100644 --- a/module/forest/leveled_forrest_test.go +++ b/module/forest/leveled_forest_test.go @@ -5,9 +5,11 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/forest/mock" + "github.com/onflow/flow-go/module/mempool" ) // ~~~~~~~~~~~~~~~~~~~~~ Mock implementation for Vertex ~~~~~~~~~~~~~~~~~~~~~ // @@ -25,12 +27,27 @@ func (v *VertexMock) Parent() (flow.Identifier, uint64) { return v.parentId, v.p func NewVertexMock(vertexId string, vertexLevel uint64, parentId string, parentLevel uint64) *mock.Vertex { v := &mock.Vertex{} - v.On("VertexID").Return(string2Identifyer(vertexId)) + v.On("VertexID").Return(string2Identifier(vertexId)) v.On("Level").Return(vertexLevel) - v.On("Parent").Return(string2Identifyer(parentId), parentLevel) + v.On("Parent").Return(string2Identifier(parentId), parentLevel) return v } +// FOREST: +// +// ↙-- [A] +// (Genesis) ← [B] ← [C] ←-- [D] +// ⋮ ⋮ ⋮ ⋮ +// ⋮ ⋮ ⋮ (Missing1) ←---- [W] +// ⋮ ⋮ ⋮ ⋮ (Missing2) ← [X] ← [Y] +// ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ↖ [Z] +// ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ ⋮ +// +// LEVEL: 0 1 2 3 4 5 6 +// Nomenclature: +// +// [B] Vertex B (internally represented as a full vertex container) +// (M) referenced vertex that has not been added (internally represented as empty vertex container) var TestVertices = map[string]*mock.Vertex{ "A": NewVertexMock("A", 3, "Genesis", 0), "B": NewVertexMock("B", 1, "Genesis", 0), @@ -77,19 +94,17 @@ func TestVertexIteratorOnEmpty(t *testing.T) { func TestLevelledForest_AddVertex(t *testing.T) { F := NewLevelledForest(0) v := NewVertexMock("A", 3, "Genesis", 0) - if err := F.VerifyVertex(v); err != nil { - assert.Fail(t, err.Error()) - } + err := F.VerifyVertex(v) + require.NoError(t, err) F.AddVertex(v) - assert.True(t, F.HasVertex(string2Identifyer("A"))) + assert.True(t, F.HasVertex(string2Identifier("A"))) // Adding Vertex twice should be fine v = NewVertexMock("A", 3, "Genesis", 0) - if err := F.VerifyVertex(v); err != nil { - assert.Fail(t, err.Error()) - } + err = F.VerifyVertex(v) + require.NoError(t, err) F.AddVertex(v) - assert.True(t, F.HasVertex(string2Identifyer("A"))) + assert.True(t, F.HasVertex(string2Identifier("A"))) } // TestLevelledForest_AcceptingGenesis checks that Levelled Forest accepts vertices @@ -99,13 +114,13 @@ func TestLevelledForest_AcceptingGenesis(t *testing.T) { // LevelledForest.LowestLevel on initial conditions F := populateNewForest(t) v1 := &mock.Vertex{} - v1.On("VertexID").Return(string2Identifyer("Root-Vertex-A_@Level0")) + v1.On("VertexID").Return(string2Identifier("Root-Vertex-A_@Level0")) v1.On("Level").Return(uint64(0)) v1.On("Parent").Return(func() (flow.Identifier, uint64) { panic("Parent() should not have been called") }) assert.NotPanics(t, func() { F.AddVertex(v1) }) v2 := &mock.Vertex{} - v2.On("VertexID").Return(string2Identifyer("Root-Vertex-B_@Level0")) + v2.On("VertexID").Return(string2Identifier("Root-Vertex-B_@Level0")) v2.On("Level").Return(uint64(0)) v2.On("Parent").Return(func() (flow.Identifier, uint64) { panic("Parent() should not have been called") }) assert.NotPanics(t, func() { F.AddVertex(v2) }) @@ -113,15 +128,15 @@ func TestLevelledForest_AcceptingGenesis(t *testing.T) { F = populateNewForest(t) err := F.PruneUpToLevel(8) // LevelledForest.LowestLevel on initial conditions - assert.True(t, err == nil) + assert.NoError(t, err) v3 := &mock.Vertex{} - v3.On("VertexID").Return(string2Identifyer("Root-Vertex-A_@Level8")) + v3.On("VertexID").Return(string2Identifier("Root-Vertex-A_@Level8")) v3.On("Level").Return(uint64(8)) v3.On("Parent").Return(func() (flow.Identifier, uint64) { panic("Parent() should not have been called") }) assert.NotPanics(t, func() { F.AddVertex(v3) }) v4 := &mock.Vertex{} - v4.On("VertexID").Return(string2Identifyer("Root-Vertex-B_@Level8")) + v4.On("VertexID").Return(string2Identifier("Root-Vertex-B_@Level8")) v4.On("Level").Return(uint64(8)) v4.On("Parent").Return(func() (flow.Identifier, uint64) { panic("Parent() should not have been called") }) assert.NotPanics(t, func() { F.AddVertex(v4) }) @@ -135,15 +150,24 @@ func TestLevelledForest_VerifyVertex(t *testing.T) { // KNOWN vertex but with wrong level number err := F.VerifyVertex(NewVertexMock("D", 10, "C", 2)) - assert.True(t, err != nil, err.Error()) + assert.Error(t, err) + assert.True(t, IsInvalidVertexError(err)) // KNOWN vertex whose PARENT references a known vertex but with mismatching level - err = F.VerifyVertex(NewVertexMock("D", 10, "C", 10)) - assert.True(t, err != nil, err.Error()) + err = F.VerifyVertex(NewVertexMock("D", 3, "C", 1)) + assert.Error(t, err) + assert.True(t, IsInvalidVertexError(err)) - // adding unknown vertex whose PARENT references a known vertex but with mismatching level + // unknown vertex whose PARENT references a known vertex but with mismatching level err = F.VerifyVertex(NewVertexMock("F", 4, "Genesis", 10)) - assert.True(t, err != nil, err.Error()) + assert.Error(t, err) + assert.True(t, IsInvalidVertexError(err)) + + // UNKNOWN vertex (Missing) that is already referenced as parent by an existing vertex [W] + // _but_ new vertex has inconsistent level compared to the parent information [W] reports + err = F.VerifyVertex(NewVertexMock("Missing1", 2, "Genesis", 0)) + assert.Error(t, err) + assert.True(t, IsInvalidVertexError(err)) } // TestLevelledForest_HasVertex test that vertices as correctly reported as contained in Forest @@ -151,12 +175,12 @@ func TestLevelledForest_VerifyVertex(t *testing.T) { // Vertices that references bvy known vertices but have not themselves are considered to be not in the tree. func TestLevelledForest_HasVertex(t *testing.T) { F := populateNewForest(t) - assert.True(t, F.HasVertex(string2Identifyer("A"))) - assert.True(t, F.HasVertex(string2Identifyer("B"))) - assert.True(t, F.HasVertex(string2Identifyer("X"))) + assert.True(t, F.HasVertex(string2Identifier("A"))) + assert.True(t, F.HasVertex(string2Identifier("B"))) + assert.True(t, F.HasVertex(string2Identifier("X"))) - assert.False(t, F.HasVertex(string2Identifyer("Genesis"))) // Genesis block never directly added (only referenced) => unknown - assert.False(t, F.HasVertex(string2Identifyer("NotYetAdded"))) // Block never mentioned before + assert.False(t, F.HasVertex(string2Identifier("Genesis"))) // Genesis block never directly added (only referenced) => unknown + assert.False(t, F.HasVertex(string2Identifier("NotYetAdded"))) // Block never mentioned before } // TestLevelledForest_GetChildren tests that children are returned properly @@ -164,7 +188,7 @@ func TestLevelledForest_GetChildren(t *testing.T) { F := populateNewForest(t) // testing children for Block that is contained in Tree - it := F.GetChildren(string2Identifyer("X")) + it := F.GetChildren(string2Identifier("X")) expectedChildren := []*mock.Vertex{ TestVertices["Y"], TestVertices["Z"], @@ -172,7 +196,7 @@ func TestLevelledForest_GetChildren(t *testing.T) { assert.ElementsMatch(t, expectedChildren, children2List(&it)) // testing children for referenced Block that is NOT contained in Tree - it = F.GetChildren(string2Identifyer("Genesis")) + it = F.GetChildren(string2Identifier("Genesis")) expectedChildren = []*mock.Vertex{ TestVertices["A"], TestVertices["B"], @@ -180,7 +204,7 @@ func TestLevelledForest_GetChildren(t *testing.T) { assert.ElementsMatch(t, expectedChildren, children2List(&it)) // testing children for Block that is contained in Tree but no children are known - it = F.GetChildren(string2Identifyer("D")) + it = F.GetChildren(string2Identifier("D")) assert.False(t, it.HasNext()) } @@ -189,13 +213,13 @@ func TestLevelledForest_GetNumberOfChildren(t *testing.T) { F := populateNewForest(t) // testing children for Block that is contained in Tree - assert.Equal(t, 2, F.GetNumberOfChildren(string2Identifyer("X"))) + assert.Equal(t, 2, F.GetNumberOfChildren(string2Identifier("X"))) // testing children for referenced Block that is NOT contained in Tree - assert.Equal(t, 2, F.GetNumberOfChildren(string2Identifyer("Genesis"))) + assert.Equal(t, 2, F.GetNumberOfChildren(string2Identifier("Genesis"))) // testing children for Block that is contained in Tree but no children are known - assert.Equal(t, 0, F.GetNumberOfChildren(string2Identifyer("D"))) + assert.Equal(t, 0, F.GetNumberOfChildren(string2Identifier("D"))) } // TestLevelledForest_GetVerticesAtLevel tests that Vertex blob is returned properly @@ -246,15 +270,15 @@ func TestLevelledForest_GetNumberOfVerticesAtLevel(t *testing.T) { // TestLevelledForest_GetVertex tests that Vertex blob is returned properly func TestLevelledForest_GetVertex(t *testing.T) { F := populateNewForest(t) - v, exists := F.GetVertex(string2Identifyer("D")) + v, exists := F.GetVertex(string2Identifier("D")) assert.Equal(t, TestVertices["D"], v) assert.True(t, exists) - v, exists = F.GetVertex(string2Identifyer("X")) + v, exists = F.GetVertex(string2Identifier("X")) assert.Equal(t, TestVertices["X"], v) assert.True(t, exists) - v, exists = F.GetVertex(string2Identifyer("Genesis")) + v, exists = F.GetVertex(string2Identifier("Genesis")) assert.Equal(t, (Vertex)(nil), v) assert.False(t, exists) } @@ -315,47 +339,54 @@ func TestLevelledForest_GetSize_DuplicatedNodes(t *testing.T) { func TestLevelledForest_PruneAtLevel(t *testing.T) { F := populateNewForest(t) err := F.PruneUpToLevel(1) - assert.False(t, err != nil) - assert.False(t, F.HasVertex(string2Identifyer("Genesis"))) - assert.True(t, F.HasVertex(string2Identifyer("A"))) - assert.True(t, F.HasVertex(string2Identifyer("B"))) - assert.True(t, F.HasVertex(string2Identifyer("C"))) - assert.True(t, F.HasVertex(string2Identifyer("D"))) - assert.True(t, F.HasVertex(string2Identifyer("X"))) - assert.True(t, F.HasVertex(string2Identifyer("Y"))) - assert.True(t, F.HasVertex(string2Identifyer("Z"))) + assert.NoError(t, err) + assert.False(t, F.HasVertex(string2Identifier("Genesis"))) + assert.True(t, F.HasVertex(string2Identifier("A"))) + assert.True(t, F.HasVertex(string2Identifier("B"))) + assert.True(t, F.HasVertex(string2Identifier("C"))) + assert.True(t, F.HasVertex(string2Identifier("D"))) + assert.True(t, F.HasVertex(string2Identifier("X"))) + assert.True(t, F.HasVertex(string2Identifier("Y"))) + assert.True(t, F.HasVertex(string2Identifier("Z"))) err = F.PruneUpToLevel(3) - assert.False(t, err != nil) - assert.False(t, F.HasVertex(string2Identifyer("Genesis"))) - assert.True(t, F.HasVertex(string2Identifyer("A"))) - assert.False(t, F.HasVertex(string2Identifyer("B"))) - assert.False(t, F.HasVertex(string2Identifyer("C"))) - assert.True(t, F.HasVertex(string2Identifyer("D"))) - assert.True(t, F.HasVertex(string2Identifyer("X"))) - assert.True(t, F.HasVertex(string2Identifyer("Y"))) - assert.True(t, F.HasVertex(string2Identifyer("Z"))) + assert.NoError(t, err) + assert.False(t, F.HasVertex(string2Identifier("Genesis"))) + assert.True(t, F.HasVertex(string2Identifier("A"))) + assert.False(t, F.HasVertex(string2Identifier("B"))) + assert.False(t, F.HasVertex(string2Identifier("C"))) + assert.True(t, F.HasVertex(string2Identifier("D"))) + assert.True(t, F.HasVertex(string2Identifier("X"))) + assert.True(t, F.HasVertex(string2Identifier("Y"))) + assert.True(t, F.HasVertex(string2Identifier("Z"))) err = F.PruneUpToLevel(6) - assert.False(t, err != nil) - assert.False(t, F.HasVertex(string2Identifyer("Genesis"))) - assert.False(t, F.HasVertex(string2Identifyer("A"))) - assert.False(t, F.HasVertex(string2Identifyer("B"))) - assert.False(t, F.HasVertex(string2Identifyer("C"))) - assert.False(t, F.HasVertex(string2Identifyer("D"))) - assert.False(t, F.HasVertex(string2Identifyer("X"))) - assert.True(t, F.HasVertex(string2Identifyer("Y"))) - assert.True(t, F.HasVertex(string2Identifyer("Z"))) + assert.NoError(t, err) + assert.False(t, F.HasVertex(string2Identifier("Genesis"))) + assert.False(t, F.HasVertex(string2Identifier("A"))) + assert.False(t, F.HasVertex(string2Identifier("B"))) + assert.False(t, F.HasVertex(string2Identifier("C"))) + assert.False(t, F.HasVertex(string2Identifier("D"))) + assert.False(t, F.HasVertex(string2Identifier("X"))) + assert.True(t, F.HasVertex(string2Identifier("Y"))) + assert.True(t, F.HasVertex(string2Identifier("Z"))) // pruning at same level repeatedly should be fine err = F.PruneUpToLevel(6) - assert.False(t, err != nil) - assert.True(t, F.HasVertex(string2Identifyer("Y"))) - assert.True(t, F.HasVertex(string2Identifyer("Z"))) + assert.NoError(t, err) + assert.True(t, F.HasVertex(string2Identifier("Y"))) + assert.True(t, F.HasVertex(string2Identifier("Z"))) // checking that pruning at lower level than what is already pruned results in error err = F.PruneUpToLevel(5) - assert.True(t, err != nil) + assert.Error(t, err) + assert.True(t, mempool.IsBelowPrunedThresholdError(err)) +} + +func TestIsInvalidVertexError(t *testing.T) { + vertex := NewVertexMock("A", 1, "B", 0) + err := NewInvalidVertexErrorf(vertex, "some error") + assert.True(t, IsInvalidVertexError(err)) } // ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Helper Functions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // @@ -382,7 +413,7 @@ func children2List(it *VertexIterator) []*mock.Vertex { return l } -func string2Identifyer(s string) flow.Identifier { +func string2Identifier(s string) flow.Identifier { var identifier flow.Identifier copy(identifier[:], []byte(s)) return identifier diff --git a/module/forest/vertex.go b/module/forest/vertex.go index 2f946b6a189..bcb090516c4 100644 --- a/module/forest/vertex.go +++ b/module/forest/vertex.go @@ -1,6 +1,9 @@ package forest import ( + "errors" + "fmt" + "github.com/onflow/flow-go/model/flow" ) @@ -9,10 +12,16 @@ type Vertex interface { VertexID() flow.Identifier // Level returns the vertex's level Level() uint64 - // Parent returns the returns the parents (level, ID) + // Parent returns the parent's (level, ID) Parent() (flow.Identifier, uint64) } +// VertexToString returns a string representation of the vertex. +func VertexToString(v Vertex) string { + parentID, parentLevel := v.Parent() + return fmt.Sprintf("<id=%x level=%d parent_id=%d parent_level=%d>", v.VertexID(), v.Level(), parentID, parentLevel) +} + // VertexIterator is a stateful iterator for VertexList. // Internally operates directly on the Vertex Containers // It has one-element look ahead for skipping empty vertex containers. @@ -53,3 +62,27 @@ func newVertexIterator(vertexList VertexList) VertexIterator { it.preLoad() return it } + +// InvalidVertexError indicates that a proposed vertex is invalid for insertion to the forest. +type InvalidVertexError struct { + // Vertex is the invalid vertex + Vertex Vertex + // msg provides additional context + msg string +} + +func (err InvalidVertexError) Error() string { + return fmt.Sprintf("invalid vertex %s: %s", VertexToString(err.Vertex), err.msg) +} + +func IsInvalidVertexError(err error) bool { + var target InvalidVertexError + return errors.As(err, &target) +} + +func NewInvalidVertexErrorf(vertex Vertex, msg string, args ...interface{}) InvalidVertexError { + return InvalidVertexError{ + Vertex: vertex, + msg: fmt.Sprintf(msg, args...), + } +} diff --git a/module/hotstuff.go b/module/hotstuff.go index 18e4c041ce3..47a7f758b6a 100644 --- a/module/hotstuff.go +++ b/module/hotstuff.go @@ -1,7 +1,7 @@ package module import ( - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/consensus/hotstuff/model" ) // HotStuff defines the interface to the core HotStuff algorithm. It includes @@ -16,7 +16,7 @@ type HotStuff interface { // // Block proposals must be submitted in order and only if they extend a // block already known to HotStuff core. - SubmitProposal(proposal *flow.Header, parentView uint64) (done <-chan struct{}) + SubmitProposal(proposal *model.Proposal) } // HotStuffFollower is run by non-consensus nodes to observe the block chain @@ -41,11 +41,12 @@ type HotStuff interface { // in addition to the proposal. type HotStuffFollower interface { ReadyDoneAware + Startable // SubmitProposal feeds a new block proposal into the HotStuffFollower. // This method blocks until the proposal is accepted to the event queue. // // Block proposals must be submitted in order, i.e. a proposal's parent must // have been previously processed by the HotStuffFollower. - SubmitProposal(proposal *flow.Header, parentView uint64) (done <-chan struct{}) + SubmitProposal(proposal *model.Proposal) } diff --git a/module/irrecoverable/irrecoverable.go b/module/irrecoverable/irrecoverable.go index c19fc4d006f..f5f428b9862 100644 --- a/module/irrecoverable/irrecoverable.go +++ b/module/irrecoverable/irrecoverable.go @@ -78,3 +78,11 @@ func Throw(ctx context.Context, err error) { // Be spectacular on how this does not -but should- handle irrecoverables: log.Fatalf("irrecoverable error signaler not found for context, please implement! Unhandled irrecoverable error %v", err) } + +// WithSignallerAndCancel returns an irrecoverable context, the cancel +// function for the context, and the error channel for the context. +func WithSignallerAndCancel(ctx context.Context) (SignalerContext, context.CancelFunc, <-chan error) { + parent, cancel := context.WithCancel(ctx) + irrecoverableCtx, errCh := WithSignaler(parent) + return irrecoverableCtx, cancel, errCh +} diff --git a/module/mempool/errors.go b/module/mempool/errors.go index 1a2dc0887fd..8c7ffbc3632 100644 --- a/module/mempool/errors.go +++ b/module/mempool/errors.go @@ -34,32 +34,29 @@ func IsUnknownExecutionResultError(err error) bool { return errors.As(err, &unknownExecutionResultError) } -// DecreasingPruningHeightError indicates that we are pruning a mempool by a height that -// is lower than existing height -type DecreasingPruningHeightError struct { +// BelowPrunedThresholdError indicates that we are attempting to query or prune a mempool by a +// key (typically block height or block view) which is lower than the lowest retained key threshold. +// In other words, we have already pruned above the specified key value. +type BelowPrunedThresholdError struct { err error } -func NewDecreasingPruningHeightError(msg string) error { - return NewDecreasingPruningHeightErrorf(msg) -} - -func NewDecreasingPruningHeightErrorf(msg string, args ...interface{}) error { - return DecreasingPruningHeightError{ +func NewBelowPrunedThresholdErrorf(msg string, args ...interface{}) error { + return BelowPrunedThresholdError{ err: fmt.Errorf(msg, args...), } } -func (e DecreasingPruningHeightError) Unwrap() error { +func (e BelowPrunedThresholdError) Unwrap() error { return e.err } -func (e DecreasingPruningHeightError) Error() string { +func (e BelowPrunedThresholdError) Error() string { return e.err.Error() } -// IsDecreasingPruningHeightError returns whether the given error is an DecreasingPruningHeightError error -func IsDecreasingPruningHeightError(err error) bool { - var newIsDecreasingPruningHeightError DecreasingPruningHeightError - return errors.As(err, &newIsDecreasingPruningHeightError) +// IsBelowPrunedThresholdError returns whether the given error is an BelowPrunedThresholdError error +func IsBelowPrunedThresholdError(err error) bool { + var newIsBelowPrunedThresholdError BelowPrunedThresholdError + return errors.As(err, &newIsBelowPrunedThresholdError) } diff --git a/module/mempool/incorporated_result_seals.go b/module/mempool/incorporated_result_seals.go index 3ef75c94e69..50e800bacac 100644 --- a/module/mempool/incorporated_result_seals.go +++ b/module/mempool/incorporated_result_seals.go @@ -37,6 +37,6 @@ type IncorporatedResultSeals interface { // Monotonicity Requirement: // The pruned height cannot decrease, as we cannot recover already pruned elements. // If `height` is smaller than the previous value, the previous value is kept - // and the sentinel empool.NewDecreasingPruningHeightError is returned. + // and the sentinel mempool.BelowPrunedThresholdError is returned. PruneUpToHeight(height uint64) error } diff --git a/module/mempool/pending_receipts.go b/module/mempool/pending_receipts.go index b26136d2a0c..2fd2d321a23 100644 --- a/module/mempool/pending_receipts.go +++ b/module/mempool/pending_receipts.go @@ -25,6 +25,6 @@ type PendingReceipts interface { // Monotonicity Requirement: // The pruned height cannot decrease, as we cannot recover already pruned elements. // If `height` is smaller than the previous value, the previous value is kept - // and the sentinel mempool.DecreasingPruningHeightError is returned. + // and the sentinel mempool.BelowPrunedThresholdError is returned. PruneUpToHeight(height uint64) error } diff --git a/module/mempool/stdmap/incorporated_result_seals.go b/module/mempool/stdmap/incorporated_result_seals.go index a030a0503e8..7202aac0f3a 100644 --- a/module/mempool/stdmap/incorporated_result_seals.go +++ b/module/mempool/stdmap/incorporated_result_seals.go @@ -152,11 +152,11 @@ func (ir *IncorporatedResultSeals) Clear() { // Monotonicity Requirement: // The pruned height cannot decrease, as we cannot recover already pruned elements. // If `height` is smaller than the previous value, the previous value is kept -// and the sentinel mempool.DecreasingPruningHeightError is returned. +// and the sentinel mempool.BelowPrunedThresholdError is returned. func (ir *IncorporatedResultSeals) PruneUpToHeight(height uint64) error { return ir.Backend.Run(func(backData mempool.BackData) error { if height < ir.lowestHeight { - return mempool.NewDecreasingPruningHeightErrorf( + return mempool.NewBelowPrunedThresholdErrorf( "pruning height: %d, existing height: %d", height, ir.lowestHeight) } diff --git a/module/mempool/stdmap/pending_receipts.go b/module/mempool/stdmap/pending_receipts.go index 8a443504382..7dc4e70e819 100644 --- a/module/mempool/stdmap/pending_receipts.go +++ b/module/mempool/stdmap/pending_receipts.go @@ -180,11 +180,11 @@ func (r *PendingReceipts) Size() uint { // Monotonicity Requirement: // The pruned height cannot decrease, as we cannot recover already pruned elements. // If `height` is smaller than the previous value, the previous value is kept -// and the sentinel mempool.DecreasingPruningHeightError is returned. +// and the sentinel mempool.BelowPrunedThresholdError is returned. func (r *PendingReceipts) PruneUpToHeight(height uint64) error { return r.Backend.Run(func(backData mempool.BackData) error { if height < r.lowestHeight { - return mempool.NewDecreasingPruningHeightErrorf( + return mempool.NewBelowPrunedThresholdErrorf( "pruning height: %d, existing height: %d", height, r.lowestHeight) } diff --git a/module/metrics.go b/module/metrics.go index 53db266f0b4..749bbbfaab1 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -3,6 +3,7 @@ package module import ( "time" + "github.com/libp2p/go-libp2p/core/peer" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/onflow/flow-go/model/chainsync" @@ -20,7 +21,7 @@ type ResolverMetrics interface { // OnDNSCacheMiss tracks the total number of dns requests resolved through looking up the network. OnDNSCacheMiss() - // DNSCacheResolution tracks the total number of dns requests resolved through the cache without + // OnDNSCacheHit tracks the total number of dns requests resolved through the cache without // looking up the network. OnDNSCacheHit() @@ -36,8 +37,8 @@ type NetworkSecurityMetrics interface { // OnUnauthorizedMessage tracks the number of unauthorized messages seen on the network. OnUnauthorizedMessage(role, msgType, topic, offense string) - // OnRateLimitedUnicastMessage tracks the number of rate limited messages seen on the network. - OnRateLimitedUnicastMessage(role, msgType, topic, reason string) + // OnRateLimitedPeer tracks the number of rate limited unicast messages seen on the network. + OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) } // GossipSubRouterMetrics encapsulates the metrics collectors for GossipSubRouter module of the networking layer. @@ -90,6 +91,7 @@ type LibP2PMetrics interface { // NetworkInboundQueueMetrics encapsulates the metrics collectors for the inbound queue of the networking layer. type NetworkInboundQueueMetrics interface { + // MessageAdded increments the metric tracking the number of messages in the queue with the given priority MessageAdded(priority int) @@ -136,10 +138,24 @@ type NetworkMetrics interface { NetworkCoreMetrics } +// EngineMetrics is a generic metrics consumer for node-internal data processing +// components (aka engines). Implementations must be non-blocking and concurrency safe. type EngineMetrics interface { + // MessageSent reports that the engine transmitted the message over the network. + // Unicasts, broadcasts, and multicasts are all reported once. MessageSent(engine string, message string) + // MessageReceived reports that the engine received the message over the network. MessageReceived(engine string, message string) + // MessageHandled reports that the engine has finished processing the message. + // Both invalid and valid messages should be reported. + // A message must be reported as either handled or dropped, not both. MessageHandled(engine string, messages string) + // InboundMessageDropped reports that the engine has dropped inbound message without processing it. + // Inbound messages must be reported as either handled or dropped, not both. + InboundMessageDropped(engine string, messages string) + // OutboundMessageDropped reports that the engine has dropped outbound message without processing it. + // Outbound messages must be reported as either sent or dropped, not both. + OutboundMessageDropped(engine string, messages string) } type ComplianceMetrics interface { @@ -148,7 +164,6 @@ type ComplianceMetrics interface { SealedHeight(height uint64) BlockFinalized(*flow.Block) BlockSealed(*flow.Block) - BlockProposalDuration(duration time.Duration) CurrentEpochCounter(counter uint64) CurrentEpochPhase(phase flow.EpochPhase) CurrentEpochFinalView(view uint64) @@ -185,26 +200,42 @@ type HotstuffMetrics interface { // HotStuffIdleDuration reports Metrics C6 HotStuff Idle Duration HotStuffIdleDuration(duration time.Duration) - // HotStuffWaitDuration reports Metrics C6 HotStuff Idle Duration + // HotStuffWaitDuration reports Metrics C6 HotStuff Idle Duration - the time between receiving and + // enqueueing a message to beginning to process that message. HotStuffWaitDuration(duration time.Duration, event string) - // SetCurView reports Metrics C8: Current View + // SetCurView reports Metrics C8: Current View maintained by Pacemaker. SetCurView(view uint64) - // SetQCView reports Metrics C9: View of Newest Known QC + // SetQCView reports Metrics C9: View of the newest QC known to Pacemaker. SetQCView(view uint64) - // CountSkipped reports the number of times we skipped ahead. + // SetTCView reports last TC known to Pacemaker. + SetTCView(view uint64) + + // CountSkipped counts the number of skips we did. CountSkipped() - // CountTimeout reports the number of times we timed out. + // CountTimeout tracks the number of views that this replica left due to observing a TC. CountTimeout() // SetTimeout sets the current timeout duration SetTimeout(duration time.Duration) + // BlockProcessingDuration measures the time which the compliance engine + // spends to process one block proposal. + BlockProcessingDuration(duration time.Duration) + + // VoteProcessingDuration measures the time which the hotstuff.VoteAggregator + // spends to process one vote. + VoteProcessingDuration(duration time.Duration) + + // TimeoutObjectProcessingDuration measures the time which the hotstuff.TimeoutAggregator + // spends to process one timeout object. + TimeoutObjectProcessingDuration(duration time.Duration) + // CommitteeProcessingDuration measures the time which the HotStuff's core logic - // spends in the hotstuff.Committee component, i.e. the time determining consensus + // spends in the hotstuff.Replicas component, i.e. the time determining consensus // committee relations. CommitteeProcessingDuration(duration time.Duration) @@ -419,17 +450,25 @@ type ExecutionDataRequesterMetrics interface { } type RuntimeMetrics interface { - // TransactionParsed reports the time spent parsing a single transaction + // RuntimeTransactionParsed reports the time spent parsing a single transaction RuntimeTransactionParsed(dur time.Duration) - // TransactionChecked reports the time spent checking a single transaction + // RuntimeTransactionChecked reports the time spent checking a single transaction RuntimeTransactionChecked(dur time.Duration) - // TransactionInterpreted reports the time spent interpreting a single transaction + // RuntimeTransactionInterpreted reports the time spent interpreting a single transaction RuntimeTransactionInterpreted(dur time.Duration) // RuntimeSetNumberOfAccounts Sets the total number of accounts on the network RuntimeSetNumberOfAccounts(count uint64) + + // RuntimeTransactionProgramsCacheMiss reports a programs cache miss + // during transaction execution + RuntimeTransactionProgramsCacheMiss() + + // RuntimeTransactionProgramsCacheHit reports a programs cache hit + // during transaction execution + RuntimeTransactionProgramsCacheHit() } type ProviderMetrics interface { @@ -528,6 +567,9 @@ type ExecutionMetrics interface { // ExecutionBlockExecutionEffortVectorComponent reports the unweighted effort of given ComputationKind at block level ExecutionBlockExecutionEffortVectorComponent(string, uint) + // ExecutionBlockCachedPrograms reports the number of cached programs at the end of a block + ExecutionBlockCachedPrograms(programs int) + // ExecutionCollectionExecuted reports the total time and computation spent on executing a collection ExecutionCollectionExecuted(dur time.Duration, stats ExecutionResultStats) diff --git a/module/metrics/chainsync.go b/module/metrics/chainsync.go index 4e18fd280b1..b58718ce81d 100644 --- a/module/metrics/chainsync.go +++ b/module/metrics/chainsync.go @@ -7,9 +7,11 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/onflow/flow-go/model/chainsync" + "github.com/onflow/flow-go/model/flow" ) type ChainSyncCollector struct { + chainID flow.ChainID timeToPruned *prometheus.HistogramVec timeToReceived *prometheus.HistogramVec totalPruned *prometheus.CounterVec @@ -18,8 +20,9 @@ type ChainSyncCollector struct { totalIdsRequested prometheus.Counter } -func NewChainSyncCollector() *ChainSyncCollector { +func NewChainSyncCollector(chainID flow.ChainID) *ChainSyncCollector { return &ChainSyncCollector{ + chainID: chainID, timeToPruned: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "time_to_pruned_seconds", Namespace: namespaceChainsync, diff --git a/module/metrics/compliance.go b/module/metrics/compliance.go index c157cc08892..baa9c9b2435 100644 --- a/module/metrics/compliance.go +++ b/module/metrics/compliance.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" ) type ComplianceCollector struct { @@ -14,7 +15,6 @@ type ComplianceCollector struct { sealedHeight prometheus.Gauge finalizedBlocks *prometheus.CounterVec sealedBlocks prometheus.Counter - blockProposalDuration prometheus.Counter finalizedPayload *prometheus.CounterVec sealedPayload *prometheus.CounterVec lastBlockFinalizedAt time.Time @@ -29,6 +29,8 @@ type ComplianceCollector struct { epochEmergencyFallbackTriggered prometheus.Gauge } +var _ module.ComplianceMetrics = (*ComplianceCollector)(nil) + func NewComplianceCollector() *ComplianceCollector { cc := &ComplianceCollector{ @@ -109,13 +111,6 @@ func NewComplianceCollector() *ComplianceCollector { Help: "the number of sealed blocks", }), - blockProposalDuration: promauto.NewCounter(prometheus.CounterOpts{ - Name: "consensus_committee_block_proposal_duration_seconds_total", - Namespace: namespaceConsensus, - Subsystem: subsystemCompliance, - Help: "time spent processing block proposals in seconds", - }), - finalizedPayload: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "finalized_payload_total", Namespace: namespaceConsensus, @@ -188,10 +183,6 @@ func (cc *ComplianceCollector) BlockSealed(block *flow.Block) { cc.sealedPayload.With(prometheus.Labels{LabelResource: ResourceSeal}).Add(float64(len(block.Payload.Seals))) } -func (cc *ComplianceCollector) BlockProposalDuration(duration time.Duration) { - cc.blockProposalDuration.Add(duration.Seconds()) -} - func (cc *ComplianceCollector) CommittedEpochFinalView(view uint64) { cc.committedEpochFinalView.Set(float64(view)) } diff --git a/module/metrics/engine.go b/module/metrics/engine.go index 16e9cf834c9..b846e75dcdf 100644 --- a/module/metrics/engine.go +++ b/module/metrics/engine.go @@ -1,16 +1,22 @@ package metrics import ( + "github.com/onflow/flow-go/module" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) type EngineCollector struct { - sent *prometheus.CounterVec - received *prometheus.CounterVec - handled *prometheus.CounterVec + sent *prometheus.CounterVec + received *prometheus.CounterVec + handled *prometheus.CounterVec + inboundDropped *prometheus.CounterVec + outboundDropped *prometheus.CounterVec } +var _ module.EngineMetrics = (*EngineCollector)(nil) + func NewEngineCollector() *EngineCollector { ec := &EngineCollector{ @@ -35,6 +41,20 @@ func NewEngineCollector() *EngineCollector { Subsystem: subsystemEngine, Help: "the number of messages handled by engines", }, []string{EngineLabel, LabelMessage}), + + inboundDropped: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "inbound_messages_dropped_total", + Namespace: namespaceNetwork, + Subsystem: subsystemEngine, + Help: "the number of inbound messages dropped by engines", + }, []string{EngineLabel, LabelMessage}), + + outboundDropped: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "outbound_messages_dropped_total", + Namespace: namespaceNetwork, + Subsystem: subsystemEngine, + Help: "the number of outbound messages dropped by engines", + }, []string{EngineLabel, LabelMessage}), } return ec @@ -51,3 +71,11 @@ func (ec *EngineCollector) MessageReceived(engine string, message string) { func (ec *EngineCollector) MessageHandled(engine string, message string) { ec.handled.With(prometheus.Labels{EngineLabel: engine, LabelMessage: message}).Inc() } + +func (ec *EngineCollector) InboundMessageDropped(engine string, message string) { + ec.inboundDropped.With(prometheus.Labels{EngineLabel: engine, LabelMessage: message}).Inc() +} + +func (ec *EngineCollector) OutboundMessageDropped(engine string, message string) { + ec.outboundDropped.With(prometheus.Labels{EngineLabel: engine, LabelMessage: message}).Inc() +} diff --git a/module/metrics/example/collection/main.go b/module/metrics/example/collection/main.go index 256de61c69f..f265e991dfa 100644 --- a/module/metrics/example/collection/main.go +++ b/module/metrics/example/collection/main.go @@ -9,8 +9,8 @@ import ( "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/metrics/example" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/queue" "github.com/onflow/flow-go/utils/unittest" ) @@ -33,15 +33,15 @@ func main() { topic1 := channels.TestNetworkChannel.String() topic2 := channels.TestMetricsChannel.String() - protocol1 := network.ProtocolTypeUnicast.String() - protocol2 := network.ProtocolTypePubSub.String() + protocol1 := message.ProtocolTypeUnicast.String() + protocol2 := message.ProtocolTypePubSub.String() message1 := "CollectionRequest" message2 := "ClusterBlockProposal" for i := 0; i < 100; i++ { collector.TransactionIngested(unittest.IdentifierFixture()) - collector.HotStuffBusyDuration(10, metrics.HotstuffEventTypeTimeout) - collector.HotStuffWaitDuration(10, metrics.HotstuffEventTypeTimeout) + collector.HotStuffBusyDuration(10, metrics.HotstuffEventTypeLocalTimeout) + collector.HotStuffWaitDuration(10, metrics.HotstuffEventTypeLocalTimeout) collector.HotStuffIdleDuration(10) collector.SetCurView(uint64(i)) collector.SetQCView(uint64(i)) diff --git a/module/metrics/example/consensus/main.go b/module/metrics/example/consensus/main.go index d86f6795260..829dfee562a 100644 --- a/module/metrics/example/consensus/main.go +++ b/module/metrics/example/consensus/main.go @@ -12,8 +12,8 @@ import ( "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/metrics/example" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/utils/unittest" ) @@ -41,8 +41,8 @@ func main() { block := unittest.BlockFixture() collector.MempoolEntries(metrics.ResourceGuarantee, 22) collector.BlockFinalized(&block) - collector.HotStuffBusyDuration(10, metrics.HotstuffEventTypeTimeout) - collector.HotStuffWaitDuration(10, metrics.HotstuffEventTypeTimeout) + collector.HotStuffBusyDuration(10, metrics.HotstuffEventTypeLocalTimeout) + collector.HotStuffWaitDuration(10, metrics.HotstuffEventTypeLocalTimeout) collector.HotStuffIdleDuration(10) collector.SetCurView(uint64(i)) collector.SetQCView(uint64(i)) @@ -66,8 +66,8 @@ func main() { collProvider := channels.TestNetworkChannel.String() collIngest := channels.TestMetricsChannel.String() - protocol1 := network.ProtocolTypeUnicast.String() - protocol2 := network.ProtocolTypePubSub.String() + protocol1 := message.ProtocolTypeUnicast.String() + protocol2 := message.ProtocolTypePubSub.String() message1 := "CollectionRequest" message2 := "ClusterBlockProposal" diff --git a/module/metrics/execution.go b/module/metrics/execution.go index 3ed3bfe6e19..2912e842472 100644 --- a/module/metrics/execution.go +++ b/module/metrics/execution.go @@ -39,6 +39,7 @@ type ExecutionCollector struct { readDurationPerValue prometheus.Histogram blockComputationUsed prometheus.Histogram blockComputationVector *prometheus.GaugeVec + blockCachedPrograms prometheus.Gauge blockMemoryUsed prometheus.Histogram blockEventCounts prometheus.Histogram blockEventSize prometheus.Histogram @@ -71,6 +72,8 @@ type ExecutionCollector struct { scriptMemoryEstimate prometheus.Histogram scriptMemoryDifference prometheus.Histogram numberOfAccounts prometheus.Gauge + programsCacheMiss prometheus.Counter + programsCacheHit prometheus.Counter chunkDataPackRequestProcessedTotal prometheus.Counter chunkDataPackProofSize prometheus.Histogram chunkDataPackCollectionSize prometheus.Histogram @@ -254,6 +257,13 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { Help: "execution effort vector of the last executed block by computation kind", }, []string{LabelComputationKind}) + blockCachedPrograms := promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespaceExecution, + Subsystem: subsystemRuntime, + Name: "block_execution_cached_programs", + Help: "Number of cached programs at the end of block execution", + }) + blockTransactionCounts := promauto.NewHistogram(prometheus.HistogramOpts{ Namespace: namespaceExecution, Subsystem: subsystemRuntime, @@ -543,6 +553,7 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { blockExecutionTime: blockExecutionTime, blockComputationUsed: blockComputationUsed, blockComputationVector: blockComputationVector, + blockCachedPrograms: blockCachedPrograms, blockMemoryUsed: blockMemoryUsed, blockEventCounts: blockEventCounts, blockEventSize: blockEventSize, @@ -650,6 +661,20 @@ func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { Help: "the number of existing accounts on the network", }), + programsCacheMiss: promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceExecution, + Subsystem: subsystemRuntime, + Name: "programs_cache_miss", + Help: "the number of times a program was not found in the cache and had to be loaded", + }), + + programsCacheHit: promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespaceExecution, + Subsystem: subsystemRuntime, + Name: "programs_cache_hit", + Help: "the number of times a program was found in the cache", + }), + maxCollectionHeight: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "max_collection_height", Namespace: namespaceExecution, @@ -706,6 +731,10 @@ func (ec *ExecutionCollector) ExecutionBlockExecutionEffortVectorComponent(compK ec.blockComputationVector.With(prometheus.Labels{LabelComputationKind: compKind}).Set(float64(value)) } +func (ec *ExecutionCollector) ExecutionBlockCachedPrograms(programs int) { + ec.blockCachedPrograms.Set(float64(programs)) +} + // TransactionExecuted reports stats for executing a transaction func (ec *ExecutionCollector) ExecutionTransactionExecuted( dur time.Duration, @@ -892,6 +921,14 @@ func (ec *ExecutionCollector) RuntimeSetNumberOfAccounts(count uint64) { ec.numberOfAccounts.Set(float64(count)) } +func (ec *ExecutionCollector) RuntimeTransactionProgramsCacheMiss() { + ec.programsCacheMiss.Inc() +} + +func (ec *ExecutionCollector) RuntimeTransactionProgramsCacheHit() { + ec.programsCacheHit.Inc() +} + func (ec *ExecutionCollector) UpdateCollectionMaxHeight(height uint64) { ec.maxCollectionHeight.Set(float64(height)) } diff --git a/module/metrics/hotstuff.go b/module/metrics/hotstuff.go index a5347a62008..258c15ddec0 100644 --- a/module/metrics/hotstuff.go +++ b/module/metrics/hotstuff.go @@ -7,14 +7,16 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" ) // HotStuff Metrics const ( - HotstuffEventTypeTimeout = "timeout" - HotstuffEventTypeOnProposal = "onproposal" - HotstuffEventTypeOnVote = "onvote" - HotstuffEventTypeOnQC = "onqc" + HotstuffEventTypeLocalTimeout = "localtimeout" + HotstuffEventTypeOnProposal = "onproposal" + HotstuffEventTypeOnQC = "onqc" + HotstuffEventTypeOnTC = "ontc" + HotstuffEventTypeOnPartialTc = "onpartialtc" ) // HotstuffCollector implements only the metrics emitted by the HotStuff core logic. @@ -30,15 +32,21 @@ type HotstuffCollector struct { waitDuration *prometheus.HistogramVec curView prometheus.Gauge qcView prometheus.Gauge + tcView prometheus.Gauge skips prometheus.Counter timeouts prometheus.Counter timeoutDuration prometheus.Gauge + voteProcessingDuration prometheus.Histogram + timeoutProcessingDuration prometheus.Histogram + blockProcessingDuration prometheus.Histogram committeeComputationsDuration prometheus.Histogram signerComputationsDuration prometheus.Histogram validatorComputationsDuration prometheus.Histogram payloadProductionDuration prometheus.Histogram } +var _ module.HotstuffMetrics = (*HotstuffCollector)(nil) + func NewHotstuffCollector(chain flow.ChainID) *HotstuffCollector { hc := &HotstuffCollector{ @@ -82,7 +90,15 @@ func NewHotstuffCollector(chain flow.ChainID) *HotstuffCollector { Name: "qc_view", Namespace: namespaceConsensus, Subsystem: subsystemHotstuff, - Help: "The view of the newest known qc from HotStuff", + Help: "The view of the newest known QC from HotStuff", + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }), + + tcView: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "tc_view", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "The view of the newest known TC from HotStuff", ConstLabels: prometheus.Labels{LabelChain: chain.String()}, }), @@ -98,7 +114,7 @@ func NewHotstuffCollector(chain flow.ChainID) *HotstuffCollector { Name: "timeouts_total", Namespace: namespaceConsensus, Subsystem: subsystemHotstuff, - Help: "The number of times we timed out during a view", + Help: "The number of views that this replica left due to observing a TC", ConstLabels: prometheus.Labels{LabelChain: chain.String()}, }), @@ -145,6 +161,30 @@ func NewHotstuffCollector(chain flow.ChainID) *HotstuffCollector { Buckets: []float64{0.02, 0.05, 0.1, 0.2, 0.5, 1, 2}, ConstLabels: prometheus.Labels{LabelChain: chain.String()}, }), + blockProcessingDuration: promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "block_processing_seconds", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "duration [seconds; measured with float64 precision] of how long compliance engine processes one block", + Buckets: []float64{0.02, 0.05, 0.1, 0.2, 0.5, 1, 2}, + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }), + voteProcessingDuration: promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "vote_processing_seconds", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "duration [seconds; measured with float64 precision] of how long VoteAggregator processes one message", + Buckets: []float64{0.02, 0.05, 0.1, 0.2, 0.5, 1, 2}, + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }), + timeoutProcessingDuration: promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "timeout_object_processing_seconds", + Namespace: namespaceConsensus, + Subsystem: subsystemHotstuff, + Help: "duration [seconds; measured with float64 precision] of how long TimeoutAggregator processes one message", + Buckets: []float64{0.02, 0.05, 0.1, 0.2, 0.5, 1, 2}, + ConstLabels: prometheus.Labels{LabelChain: chain.String()}, + }), } return hc @@ -160,29 +200,51 @@ func (hc *HotstuffCollector) HotStuffIdleDuration(duration time.Duration) { hc.idleDuration.Observe(duration.Seconds()) // unit: seconds; with float64 precision } -// HotStuffWaitDuration reports Metrics C6 HotStuff Wait Duration +// HotStuffWaitDuration reports Metrics C6 HotStuff Idle Duration - the time between receiving and +// enqueueing a message to beginning to process that message. func (hc *HotstuffCollector) HotStuffWaitDuration(duration time.Duration, event string) { hc.waitDuration.WithLabelValues(event).Observe(duration.Seconds()) // unit: seconds; with float64 precision } -// HotstuffCollector reports Metrics C8: Current View +// CountSkipped counts the number of skips we did. +func (hc *HotstuffCollector) CountSkipped() { + hc.skips.Inc() +} + +// CountTimeout tracks the number of views that this replica left due to observing a TC. +func (hc *HotstuffCollector) CountTimeout() { + hc.timeouts.Inc() +} + +// SetCurView reports Metrics C8: Current View func (hc *HotstuffCollector) SetCurView(view uint64) { hc.curView.Set(float64(view)) } -// NewestKnownQC reports Metrics C9: View of Newest Known QC +// SetQCView reports Metrics C9: View of Newest Known QC func (hc *HotstuffCollector) SetQCView(view uint64) { hc.qcView.Set(float64(view)) } -// CountSkipped counts the number of skips we did. -func (hc *HotstuffCollector) CountSkipped() { - hc.skips.Inc() +// SetTCView reports the view of the newest known TC +func (hc *HotstuffCollector) SetTCView(view uint64) { + hc.tcView.Set(float64(view)) } -// CountTimeout counts the number of timeouts we had. -func (hc *HotstuffCollector) CountTimeout() { - hc.timeouts.Inc() +// BlockProcessingDuration measures the time which the compliance engine +// spends to process one block proposal. +func (hc *HotstuffCollector) BlockProcessingDuration(duration time.Duration) { + hc.blockProcessingDuration.Observe(duration.Seconds()) +} + +// VoteProcessingDuration reports the processing time for a single vote +func (hc *HotstuffCollector) VoteProcessingDuration(duration time.Duration) { + hc.voteProcessingDuration.Observe(duration.Seconds()) +} + +// TimeoutObjectProcessingDuration reports the processing time for a TimeoutObject +func (hc *HotstuffCollector) TimeoutObjectProcessingDuration(duration time.Duration) { + hc.timeoutProcessingDuration.Observe(duration.Seconds()) } // SetTimeout sets the current timeout duration. diff --git a/module/metrics/hotstuff/consumer.go b/module/metrics/hotstuff/consumer.go index a0fbd478591..0b5e0abf168 100644 --- a/module/metrics/hotstuff/consumer.go +++ b/module/metrics/hotstuff/consumer.go @@ -1,6 +1,7 @@ package consensus import ( + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/notifications" "github.com/onflow/flow-go/model/flow" @@ -16,28 +17,26 @@ type MetricsConsumer struct { metrics module.HotstuffMetrics } +var _ hotstuff.Consumer = (*MetricsConsumer)(nil) + func NewMetricsConsumer(metrics module.HotstuffMetrics) *MetricsConsumer { return &MetricsConsumer{ metrics: metrics, } } -func (c *MetricsConsumer) OnEnteringView(view uint64, leader flow.Identifier) { - c.metrics.SetCurView(view) -} - -func (c *MetricsConsumer) OnQcIncorporated(qc *flow.QuorumCertificate) { +func (c *MetricsConsumer) OnQcTriggeredViewChange(_ uint64, newView uint64, qc *flow.QuorumCertificate) { + c.metrics.SetCurView(newView) c.metrics.SetQCView(qc.View) -} - -func (c *MetricsConsumer) OnQcTriggeredViewChange(qc *flow.QuorumCertificate, newView uint64) { c.metrics.CountSkipped() } -func (c *MetricsConsumer) OnReachedTimeout(info *model.TimerInfo) { +func (c *MetricsConsumer) OnTcTriggeredViewChange(_ uint64, newView uint64, tc *flow.TimeoutCertificate) { + c.metrics.SetCurView(newView) + c.metrics.SetTCView(tc.View) c.metrics.CountTimeout() } -func (c *MetricsConsumer) OnStartingTimeout(info *model.TimerInfo) { +func (c *MetricsConsumer) OnStartingTimeout(info model.TimerInfo) { c.metrics.SetTimeout(info.Duration) } diff --git a/module/metrics/labels.go b/module/metrics/labels.go index 05a15503658..ba9aa4c1b0e 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -21,19 +21,22 @@ const ( const ( // collection - EngineClusterCompliance = "proposal" + EngineClusterCompliance = "collection_compliance" + EngineCollectionMessageHub = "collection_message_hub" EngineCollectionIngest = "collection_ingest" EngineCollectionProvider = "collection_provider" EngineClusterSynchronization = "cluster-sync" // consensus - EnginePropagation = "propagation" - EngineCompliance = "compliance" - EngineConsensusProvider = "consensus_provider" - EngineConsensusIngestion = "consensus_ingestion" - EngineSealing = "sealing" - EngineSynchronization = "sync" + EnginePropagation = "propagation" + EngineCompliance = "compliance" + EngineConsensusMessageHub = "consensus_message_hub" + EngineConsensusIngestion = "consensus_ingestion" + EngineSealing = "sealing" + EngineSynchronization = "sync" // common - EngineFollower = "follower" + EngineFollower = "follower" + EngineVoteAggregator = "vote_aggregator" + EngineTimeoutAggregator = "timeout_aggregator" ) const ( @@ -80,7 +83,6 @@ const ( ResourceNetworkingDnsTxtCache = "networking_dns_txt_cache" // networking layer ResourceClusterBlockProposalQueue = "cluster_compliance_proposal_queue" // collection node, compliance engine - ResourceClusterBlockVoteQueue = "cluster_compliance_vote_queue" // collection node, compliance engine ResourceTransactionIngestQueue = "ingest_transaction_queue" // collection node, ingest engine ResourceBeaconKey = "beacon-key" // consensus node, DKG engine ResourceApprovalQueue = "sealing_approval_queue" // consensus node, sealing engine @@ -88,7 +90,8 @@ const ( ResourceApprovalResponseQueue = "sealing_approval_response_queue" // consensus node, sealing engine ResourceBlockResponseQueue = "compliance_block_response_queue" // consensus node, compliance engine ResourceBlockProposalQueue = "compliance_proposal_queue" // consensus node, compliance engine - ResourceBlockVoteQueue = "compliance_vote_queue" // consensus node, compliance engine + ResourceBlockVoteQueue = "vote_aggregator_queue" // consensus/collection node, vote aggregator + ResourceTimeoutObjectQueue = "timeout_aggregator_queue" // consensus/collection node, timeout aggregator ResourceCollectionGuaranteesQueue = "ingestion_col_guarantee_queue" // consensus node, ingestion engine ResourceChunkDataPack = "chunk_data_pack" // execution node ResourceChunkDataPackRequests = "chunk_data_pack_request" // execution node @@ -100,27 +103,25 @@ const ( ) const ( - MessageCollectionGuarantee = "guarantee" - MessageBlockProposal = "proposal" - MessageBlockVote = "vote" - MessageExecutionReceipt = "receipt" - MessageResultApproval = "approval" - MessageSyncRequest = "ping" - MessageSyncResponse = "pong" - MessageRangeRequest = "range" - MessageBatchRequest = "batch" - MessageBlockResponse = "block" - MessageSyncedBlock = "synced_block" - MessageClusterBlockProposal = "cluster_proposal" - MessageClusterBlockVote = "cluster_vote" - MessageClusterBlockResponse = "cluster_block_response" - MessageSyncedClusterBlock = "synced_cluster_block" - MessageTransaction = "transaction" - MessageSubmitGuarantee = "submit_guarantee" - MessageCollectionRequest = "collection_request" - MessageCollectionResponse = "collection_response" - MessageEntityRequest = "entity_request" - MessageEntityResponse = "entity_response" + MessageCollectionGuarantee = "guarantee" + MessageBlockProposal = "proposal" + MessageBlockVote = "vote" + MessageTimeoutObject = "timeout_object" + MessageExecutionReceipt = "receipt" + MessageResultApproval = "approval" + MessageSyncRequest = "ping" + MessageSyncResponse = "pong" + MessageRangeRequest = "range" + MessageBatchRequest = "batch" + MessageBlockResponse = "block" + MessageSyncedBlock = "synced_block" + MessageSyncedClusterBlock = "synced_cluster_block" + MessageTransaction = "transaction" + MessageSubmitGuarantee = "submit_guarantee" + MessageCollectionRequest = "collection_request" + MessageCollectionResponse = "collection_response" + MessageEntityRequest = "entity_request" + MessageEntityResponse = "entity_response" ) const ExecutionDataRequestRetryable = "retryable" diff --git a/module/metrics/network.go b/module/metrics/network.go index 93151ec64ab..0f7d8f75351 100644 --- a/module/metrics/network.go +++ b/module/metrics/network.go @@ -4,11 +4,13 @@ import ( "strconv" "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/rs/zerolog" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/utils/logging" ) const ( @@ -334,7 +336,15 @@ func (nc *NetworkCollector) OnUnauthorizedMessage(role, msgType, topic, offense nc.unAuthorizedMessagesCount.WithLabelValues(role, msgType, topic, offense).Inc() } -// OnRateLimitedUnicastMessage tracks the number of rate limited messages seen on the network. -func (nc *NetworkCollector) OnRateLimitedUnicastMessage(role, msgType, topic, reason string) { +// OnRateLimitedPeer tracks the number of rate limited messages seen on the network. +func (nc *NetworkCollector) OnRateLimitedPeer(peerID peer.ID, role, msgType, topic, reason string) { + nc.logger.Warn(). + Str("peer_id", peerID.String()). + Str("role", role). + Str("message_type", msgType). + Str("topic", topic). + Str("reason", reason). + Bool(logging.KeySuspicious, true). + Msg("unicast peer rate limited") nc.rateLimitedUnicastMessagesCount.WithLabelValues(role, msgType, topic, reason).Inc() } diff --git a/module/metrics/noop.go b/module/metrics/noop.go index bb166d291f1..7f5f7c96d50 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "time" "github.com/libp2p/go-libp2p/core/network" @@ -11,98 +12,122 @@ import ( "github.com/onflow/flow-go/model/cluster" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + + httpmetrics "github.com/slok/go-http-metrics/metrics" ) type NoopCollector struct{} -func (nc *NoopCollector) Peers(prefix string, n int) {} -func (nc *NoopCollector) Wantlist(prefix string, n int) {} -func (nc *NoopCollector) BlobsReceived(prefix string, n uint64) {} -func (nc *NoopCollector) DataReceived(prefix string, n uint64) {} -func (nc *NoopCollector) BlobsSent(prefix string, n uint64) {} -func (nc *NoopCollector) DataSent(prefix string, n uint64) {} -func (nc *NoopCollector) DupBlobsReceived(prefix string, n uint64) {} -func (nc *NoopCollector) DupDataReceived(prefix string, n uint64) {} -func (nc *NoopCollector) MessagesReceived(prefix string, n uint64) {} -func (nc *NoopCollector) OutboundMessageSent(int, string, string, string) {} -func (nc *NoopCollector) InboundMessageReceived(int, string, string, string) {} -func (nc *NoopCollector) DuplicateInboundMessagesDropped(string, string, string) {} -func (nc *NoopCollector) MessageAdded(priority int) {} -func (nc *NoopCollector) MessageRemoved(priority int) {} -func (nc *NoopCollector) QueueDuration(duration time.Duration, priority int) {} -func (nc *NoopCollector) MessageProcessingStarted(topic string) {} -func (nc *NoopCollector) MessageProcessingFinished(topic string, duration time.Duration) {} -func (nc *NoopCollector) UnicastMessageSendingStarted(topic string) {} -func (nc *NoopCollector) UnicastMessageSendingCompleted(topic string) {} -func (nc *NoopCollector) MessageSent(engine string, message string) {} -func (nc *NoopCollector) MessageReceived(engine string, message string) {} -func (nc *NoopCollector) MessageHandled(engine string, message string) {} -func (nc *NoopCollector) OutboundConnections(_ uint) {} -func (nc *NoopCollector) InboundConnections(_ uint) {} -func (nc *NoopCollector) DNSLookupDuration(duration time.Duration) {} -func (nc *NoopCollector) OnDNSCacheMiss() {} -func (nc *NoopCollector) OnDNSCacheInvalidated() {} -func (nc *NoopCollector) OnDNSCacheHit() {} -func (nc *NoopCollector) OnDNSLookupRequestDropped() {} -func (nc *NoopCollector) UnstakedOutboundConnections(_ uint) {} -func (nc *NoopCollector) UnstakedInboundConnections(_ uint) {} -func (nc *NoopCollector) RanGC(duration time.Duration) {} -func (nc *NoopCollector) BadgerLSMSize(sizeBytes int64) {} -func (nc *NoopCollector) BadgerVLogSize(sizeBytes int64) {} -func (nc *NoopCollector) BadgerNumReads(n int64) {} -func (nc *NoopCollector) BadgerNumWrites(n int64) {} -func (nc *NoopCollector) BadgerNumBytesRead(n int64) {} -func (nc *NoopCollector) BadgerNumBytesWritten(n int64) {} -func (nc *NoopCollector) BadgerNumGets(n int64) {} -func (nc *NoopCollector) BadgerNumPuts(n int64) {} -func (nc *NoopCollector) BadgerNumBlockedPuts(n int64) {} -func (nc *NoopCollector) BadgerNumMemtableGets(n int64) {} -func (nc *NoopCollector) FinalizedHeight(height uint64) {} -func (nc *NoopCollector) SealedHeight(height uint64) {} -func (nc *NoopCollector) BlockProposed(*flow.Block) {} -func (nc *NoopCollector) BlockFinalized(*flow.Block) {} -func (nc *NoopCollector) BlockSealed(*flow.Block) {} -func (nc *NoopCollector) BlockProposalDuration(duration time.Duration) {} -func (nc *NoopCollector) CommittedEpochFinalView(view uint64) {} -func (nc *NoopCollector) CurrentEpochCounter(counter uint64) {} -func (nc *NoopCollector) CurrentEpochPhase(phase flow.EpochPhase) {} -func (nc *NoopCollector) CurrentEpochFinalView(view uint64) {} -func (nc *NoopCollector) CurrentDKGPhase1FinalView(view uint64) {} -func (nc *NoopCollector) CurrentDKGPhase2FinalView(view uint64) {} -func (nc *NoopCollector) CurrentDKGPhase3FinalView(view uint64) {} -func (nc *NoopCollector) EpochEmergencyFallbackTriggered() {} -func (nc *NoopCollector) CacheEntries(resource string, entries uint) {} -func (nc *NoopCollector) CacheHit(resource string) {} -func (nc *NoopCollector) CacheNotFound(resource string) {} -func (nc *NoopCollector) CacheMiss(resource string) {} -func (nc *NoopCollector) MempoolEntries(resource string, entries uint) {} -func (nc *NoopCollector) Register(resource string, entriesFunc module.EntriesFunc) error { return nil } -func (nc *NoopCollector) HotStuffBusyDuration(duration time.Duration, event string) {} -func (nc *NoopCollector) HotStuffIdleDuration(duration time.Duration) {} -func (nc *NoopCollector) HotStuffWaitDuration(duration time.Duration, event string) {} -func (nc *NoopCollector) SetCurView(view uint64) {} -func (nc *NoopCollector) SetQCView(view uint64) {} -func (nc *NoopCollector) CountSkipped() {} -func (nc *NoopCollector) CountTimeout() {} -func (nc *NoopCollector) SetTimeout(duration time.Duration) {} -func (nc *NoopCollector) CommitteeProcessingDuration(duration time.Duration) {} -func (nc *NoopCollector) SignerProcessingDuration(duration time.Duration) {} -func (nc *NoopCollector) ValidatorProcessingDuration(duration time.Duration) {} -func (nc *NoopCollector) PayloadProductionDuration(duration time.Duration) {} -func (nc *NoopCollector) TransactionIngested(txID flow.Identifier) {} -func (nc *NoopCollector) ClusterBlockProposed(*cluster.Block) {} -func (nc *NoopCollector) ClusterBlockFinalized(*cluster.Block) {} -func (nc *NoopCollector) StartCollectionToFinalized(collectionID flow.Identifier) {} -func (nc *NoopCollector) FinishCollectionToFinalized(collectionID flow.Identifier) {} -func (nc *NoopCollector) StartBlockToSeal(blockID flow.Identifier) {} -func (nc *NoopCollector) FinishBlockToSeal(blockID flow.Identifier) {} -func (nc *NoopCollector) EmergencySeal() {} -func (nc *NoopCollector) OnReceiptProcessingDuration(duration time.Duration) {} -func (nc *NoopCollector) OnApprovalProcessingDuration(duration time.Duration) {} -func (nc *NoopCollector) CheckSealingDuration(duration time.Duration) {} -func (nc *NoopCollector) OnExecutionResultReceivedAtAssignerEngine() {} -func (nc *NoopCollector) OnVerifiableChunkReceivedAtVerifierEngine() {} -func (nc *NoopCollector) OnResultApprovalDispatchedInNetworkByVerifier() {} +func NewNoopCollector() *NoopCollector { + nc := &NoopCollector{} + return nc +} + +func (nc *NoopCollector) OutboundMessageSent(int, string, string, string) {} +func (nc *NoopCollector) InboundMessageReceived(int, string, string, string) {} +func (nc *NoopCollector) DuplicateInboundMessagesDropped(string, string, string) {} +func (nc *NoopCollector) UnicastMessageSendingStarted(topic string) {} +func (nc *NoopCollector) UnicastMessageSendingCompleted(topic string) {} +func (nc *NoopCollector) BlockProposed(*flow.Block) {} +func (nc *NoopCollector) BlockProposalDuration(duration time.Duration) {} + +var _ module.HotstuffMetrics = (*NoopCollector)(nil) +var _ module.EngineMetrics = (*NoopCollector)(nil) +var _ module.HeroCacheMetrics = (*NoopCollector)(nil) +var _ module.NetworkMetrics = (*NoopCollector)(nil) + +func (nc *NoopCollector) Peers(prefix string, n int) {} +func (nc *NoopCollector) Wantlist(prefix string, n int) {} +func (nc *NoopCollector) BlobsReceived(prefix string, n uint64) {} +func (nc *NoopCollector) DataReceived(prefix string, n uint64) {} +func (nc *NoopCollector) BlobsSent(prefix string, n uint64) {} +func (nc *NoopCollector) DataSent(prefix string, n uint64) {} +func (nc *NoopCollector) DupBlobsReceived(prefix string, n uint64) {} +func (nc *NoopCollector) DupDataReceived(prefix string, n uint64) {} +func (nc *NoopCollector) MessagesReceived(prefix string, n uint64) {} +func (nc *NoopCollector) NetworkMessageSent(sizeBytes int, topic string, messageType string) {} +func (nc *NoopCollector) NetworkMessageReceived(sizeBytes int, topic string, messageType string) {} +func (nc *NoopCollector) NetworkDuplicateMessagesDropped(topic string, messageType string) {} +func (nc *NoopCollector) MessageAdded(priority int) {} +func (nc *NoopCollector) MessageRemoved(priority int) {} +func (nc *NoopCollector) QueueDuration(duration time.Duration, priority int) {} +func (nc *NoopCollector) MessageProcessingStarted(topic string) {} +func (nc *NoopCollector) MessageProcessingFinished(topic string, duration time.Duration) {} +func (nc *NoopCollector) DirectMessageStarted(topic string) {} +func (nc *NoopCollector) DirectMessageFinished(topic string) {} +func (nc *NoopCollector) MessageSent(engine string, message string) {} +func (nc *NoopCollector) MessageReceived(engine string, message string) {} +func (nc *NoopCollector) MessageHandled(engine string, message string) {} +func (nc *NoopCollector) InboundMessageDropped(engine string, message string) {} +func (nc *NoopCollector) OutboundMessageDropped(engine string, messages string) {} +func (nc *NoopCollector) OutboundConnections(_ uint) {} +func (nc *NoopCollector) InboundConnections(_ uint) {} +func (nc *NoopCollector) DNSLookupDuration(duration time.Duration) {} +func (nc *NoopCollector) OnDNSCacheMiss() {} +func (nc *NoopCollector) OnDNSCacheInvalidated() {} +func (nc *NoopCollector) OnDNSCacheHit() {} +func (nc *NoopCollector) OnDNSLookupRequestDropped() {} +func (nc *NoopCollector) UnstakedOutboundConnections(_ uint) {} +func (nc *NoopCollector) UnstakedInboundConnections(_ uint) {} +func (nc *NoopCollector) RanGC(duration time.Duration) {} +func (nc *NoopCollector) BadgerLSMSize(sizeBytes int64) {} +func (nc *NoopCollector) BadgerVLogSize(sizeBytes int64) {} +func (nc *NoopCollector) BadgerNumReads(n int64) {} +func (nc *NoopCollector) BadgerNumWrites(n int64) {} +func (nc *NoopCollector) BadgerNumBytesRead(n int64) {} +func (nc *NoopCollector) BadgerNumBytesWritten(n int64) {} +func (nc *NoopCollector) BadgerNumGets(n int64) {} +func (nc *NoopCollector) BadgerNumPuts(n int64) {} +func (nc *NoopCollector) BadgerNumBlockedPuts(n int64) {} +func (nc *NoopCollector) BadgerNumMemtableGets(n int64) {} +func (nc *NoopCollector) FinalizedHeight(height uint64) {} +func (nc *NoopCollector) SealedHeight(height uint64) {} +func (nc *NoopCollector) BlockFinalized(*flow.Block) {} +func (nc *NoopCollector) BlockSealed(*flow.Block) {} +func (nc *NoopCollector) CommittedEpochFinalView(view uint64) {} +func (nc *NoopCollector) CurrentEpochCounter(counter uint64) {} +func (nc *NoopCollector) CurrentEpochPhase(phase flow.EpochPhase) {} +func (nc *NoopCollector) CurrentEpochFinalView(view uint64) {} +func (nc *NoopCollector) CurrentDKGPhase1FinalView(view uint64) {} +func (nc *NoopCollector) CurrentDKGPhase2FinalView(view uint64) {} +func (nc *NoopCollector) CurrentDKGPhase3FinalView(view uint64) {} +func (nc *NoopCollector) EpochEmergencyFallbackTriggered() {} +func (nc *NoopCollector) CacheEntries(resource string, entries uint) {} +func (nc *NoopCollector) CacheHit(resource string) {} +func (nc *NoopCollector) CacheNotFound(resource string) {} +func (nc *NoopCollector) CacheMiss(resource string) {} +func (nc *NoopCollector) MempoolEntries(resource string, entries uint) {} +func (nc *NoopCollector) Register(resource string, entriesFunc module.EntriesFunc) error { return nil } +func (nc *NoopCollector) HotStuffBusyDuration(duration time.Duration, event string) {} +func (nc *NoopCollector) HotStuffIdleDuration(duration time.Duration) {} +func (nc *NoopCollector) HotStuffWaitDuration(duration time.Duration, event string) {} +func (nc *NoopCollector) SetCurView(view uint64) {} +func (nc *NoopCollector) SetQCView(view uint64) {} +func (nc *NoopCollector) SetTCView(uint64) {} +func (nc *NoopCollector) CountSkipped() {} +func (nc *NoopCollector) CountTimeout() {} +func (nc *NoopCollector) BlockProcessingDuration(time.Duration) {} +func (nc *NoopCollector) VoteProcessingDuration(time.Duration) {} +func (nc *NoopCollector) TimeoutObjectProcessingDuration(time.Duration) {} +func (nc *NoopCollector) SetTimeout(duration time.Duration) {} +func (nc *NoopCollector) CommitteeProcessingDuration(duration time.Duration) {} +func (nc *NoopCollector) SignerProcessingDuration(duration time.Duration) {} +func (nc *NoopCollector) ValidatorProcessingDuration(duration time.Duration) {} +func (nc *NoopCollector) PayloadProductionDuration(duration time.Duration) {} +func (nc *NoopCollector) TransactionIngested(txID flow.Identifier) {} +func (nc *NoopCollector) ClusterBlockProposed(*cluster.Block) {} +func (nc *NoopCollector) ClusterBlockFinalized(*cluster.Block) {} +func (nc *NoopCollector) StartCollectionToFinalized(collectionID flow.Identifier) {} +func (nc *NoopCollector) FinishCollectionToFinalized(collectionID flow.Identifier) {} +func (nc *NoopCollector) StartBlockToSeal(blockID flow.Identifier) {} +func (nc *NoopCollector) FinishBlockToSeal(blockID flow.Identifier) {} +func (nc *NoopCollector) EmergencySeal() {} +func (nc *NoopCollector) OnReceiptProcessingDuration(duration time.Duration) {} +func (nc *NoopCollector) OnApprovalProcessingDuration(duration time.Duration) {} +func (nc *NoopCollector) CheckSealingDuration(duration time.Duration) {} +func (nc *NoopCollector) OnExecutionResultReceivedAtAssignerEngine() {} +func (nc *NoopCollector) OnVerifiableChunkReceivedAtVerifierEngine() {} +func (nc *NoopCollector) OnResultApprovalDispatchedInNetworkByVerifier() {} func (nc *NoopCollector) SetMaxChunkDataPackAttemptsForNextUnsealedHeightAtRequester(attempts uint64) { } func (nc *NoopCollector) OnFinalizedBlockArrivedAtAssigner(height uint64) {} @@ -134,6 +159,7 @@ func (nc *NoopCollector) ExecutionBlockExecuted(_ time.Duration, _ module.Execut func (nc *NoopCollector) ExecutionCollectionExecuted(_ time.Duration, _ module.ExecutionResultStats) { } func (nc *NoopCollector) ExecutionBlockExecutionEffortVectorComponent(_ string, _ uint) {} +func (ec *NoopCollector) ExecutionBlockCachedPrograms(programs int) {} func (nc *NoopCollector) ExecutionTransactionExecuted(_ time.Duration, _, _, _ uint64, _, _ int, _ bool) { } func (nc *NoopCollector) ExecutionChunkDataPackGenerated(_, _ int) {} @@ -161,6 +187,8 @@ func (nc *NoopCollector) RuntimeTransactionParsed(dur time.Duration) func (nc *NoopCollector) RuntimeTransactionChecked(dur time.Duration) {} func (nc *NoopCollector) RuntimeTransactionInterpreted(dur time.Duration) {} func (nc *NoopCollector) RuntimeSetNumberOfAccounts(count uint64) {} +func (nc *NoopCollector) RuntimeTransactionProgramsCacheMiss() {} +func (nc *NoopCollector) RuntimeTransactionProgramsCacheHit() {} func (nc *NoopCollector) ScriptExecuted(dur time.Duration, size int) {} func (nc *NoopCollector) TransactionResultFetched(dur time.Duration, size int) {} func (nc *NoopCollector) TransactionReceived(txID flow.Identifier, when time.Time) {} @@ -209,34 +237,38 @@ func (nc *NoopCollector) PrunedBlocks(totalByHeight, totalById, storedByHeight, func (nc *NoopCollector) RangeRequested(ran chainsync.Range) {} func (nc *NoopCollector) BatchRequested(batch chainsync.Batch) {} func (nc *NoopCollector) OnUnauthorizedMessage(role, msgType, topic, offense string) {} -func (nc *NoopCollector) OnRateLimitedUnicastMessage(role, msgType, topic, reason string) {} -func (nc *NoopCollector) OnIWantReceived(int) {} -func (nc *NoopCollector) OnIHaveReceived(int) {} -func (nc *NoopCollector) OnGraftReceived(int) {} -func (nc *NoopCollector) OnPruneReceived(int) {} -func (nc *NoopCollector) OnIncomingRpcAcceptedFully() {} -func (nc *NoopCollector) OnIncomingRpcAcceptedOnlyForControlMessages() {} -func (nc *NoopCollector) OnIncomingRpcRejected() {} -func (nc *NoopCollector) OnPublishedGossipMessagesReceived(count int) {} -func (nc *NoopCollector) AllowConn(dir network.Direction, usefd bool) {} -func (nc *NoopCollector) BlockConn(dir network.Direction, usefd bool) {} -func (nc *NoopCollector) AllowStream(p peer.ID, dir network.Direction) {} -func (nc *NoopCollector) BlockStream(p peer.ID, dir network.Direction) {} -func (nc *NoopCollector) AllowPeer(p peer.ID) {} -func (nc *NoopCollector) BlockPeer(p peer.ID) {} -func (nc *NoopCollector) AllowProtocol(proto protocol.ID) {} -func (nc *NoopCollector) BlockProtocol(proto protocol.ID) {} -func (nc *NoopCollector) BlockProtocolPeer(proto protocol.ID, p peer.ID) {} -func (nc *NoopCollector) AllowService(svc string) {} -func (nc *NoopCollector) BlockService(svc string) {} -func (nc *NoopCollector) BlockServicePeer(svc string, p peer.ID) {} -func (nc *NoopCollector) AllowMemory(size int) {} -func (nc *NoopCollector) BlockMemory(size int) {} +func (nc *NoopCollector) ObserveHTTPRequestDuration(context.Context, httpmetrics.HTTPReqProperties, time.Duration) { +} +func (nc *NoopCollector) ObserveHTTPResponseSize(context.Context, httpmetrics.HTTPReqProperties, int64) { +} +func (nc *NoopCollector) AddInflightRequests(context.Context, httpmetrics.HTTPProperties, int) {} +func (nc *NoopCollector) AddTotalRequests(context.Context, string, string) {} +func (nc *NoopCollector) OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) { +} var _ module.HeroCacheMetrics = (*NoopCollector)(nil) var _ module.NetworkMetrics = (*NoopCollector)(nil) -func NewNoopCollector() *NoopCollector { - nc := &NoopCollector{} - return nc -} +func (nc *NoopCollector) OnRateLimitedUnicastMessage(role, msgType, topic, reason string) {} +func (nc *NoopCollector) OnIWantReceived(int) {} +func (nc *NoopCollector) OnIHaveReceived(int) {} +func (nc *NoopCollector) OnGraftReceived(int) {} +func (nc *NoopCollector) OnPruneReceived(int) {} +func (nc *NoopCollector) OnIncomingRpcAcceptedFully() {} +func (nc *NoopCollector) OnIncomingRpcAcceptedOnlyForControlMessages() {} +func (nc *NoopCollector) OnIncomingRpcRejected() {} +func (nc *NoopCollector) OnPublishedGossipMessagesReceived(count int) {} +func (nc *NoopCollector) AllowConn(dir network.Direction, usefd bool) {} +func (nc *NoopCollector) BlockConn(dir network.Direction, usefd bool) {} +func (nc *NoopCollector) AllowStream(p peer.ID, dir network.Direction) {} +func (nc *NoopCollector) BlockStream(p peer.ID, dir network.Direction) {} +func (nc *NoopCollector) AllowPeer(p peer.ID) {} +func (nc *NoopCollector) BlockPeer(p peer.ID) {} +func (nc *NoopCollector) AllowProtocol(proto protocol.ID) {} +func (nc *NoopCollector) BlockProtocol(proto protocol.ID) {} +func (nc *NoopCollector) BlockProtocolPeer(proto protocol.ID, p peer.ID) {} +func (nc *NoopCollector) AllowService(svc string) {} +func (nc *NoopCollector) BlockService(svc string) {} +func (nc *NoopCollector) BlockServicePeer(svc string, p peer.ID) {} +func (nc *NoopCollector) AllowMemory(size int) {} +func (nc *NoopCollector) BlockMemory(size int) {} diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go new file mode 100644 index 00000000000..36c3d1b8b1a --- /dev/null +++ b/module/metrics/rest_api.go @@ -0,0 +1,109 @@ +package metrics + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + + httpmetrics "github.com/slok/go-http-metrics/metrics" + metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" +) + +// Example recorder taken from: +// https://github.com/slok/go-http-metrics/blob/master/metrics/prometheus/prometheus.go +type RestCollector interface { + httpmetrics.Recorder + AddTotalRequests(ctx context.Context, service string, id string) +} + +type recorder struct { + httpRequestDurHistogram *prometheus.HistogramVec + httpResponseSizeHistogram *prometheus.HistogramVec + httpRequestsInflight *prometheus.GaugeVec + httpRequestsTotal *prometheus.GaugeVec +} + +// NewRestCollector returns a new metrics recorder that implements the recorder +// using Prometheus as the backend. +func NewRestCollector(cfg metricsProm.Config) RestCollector { + if len(cfg.DurationBuckets) == 0 { + cfg.DurationBuckets = prometheus.DefBuckets + } + + if len(cfg.SizeBuckets) == 0 { + cfg.SizeBuckets = prometheus.ExponentialBuckets(100, 10, 8) + } + + if cfg.Registry == nil { + cfg.Registry = prometheus.DefaultRegisterer + } + + if cfg.HandlerIDLabel == "" { + cfg.HandlerIDLabel = "handler" + } + + if cfg.StatusCodeLabel == "" { + cfg.StatusCodeLabel = "code" + } + + if cfg.MethodLabel == "" { + cfg.MethodLabel = "method" + } + + if cfg.ServiceLabel == "" { + cfg.ServiceLabel = "service" + } + + r := &recorder{ + httpRequestDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: cfg.Prefix, + Subsystem: "http", + Name: "request_duration_seconds", + Help: "The latency of the HTTP requests.", + Buckets: cfg.DurationBuckets, + }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + + httpResponseSizeHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: cfg.Prefix, + Subsystem: "http", + Name: "response_size_bytes", + Help: "The size of the HTTP responses.", + Buckets: cfg.SizeBuckets, + }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + + httpRequestsInflight: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: cfg.Prefix, + Subsystem: "http", + Name: "requests_inflight", + Help: "The number of inflight requests being handled at the same time.", + }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + + httpRequestsTotal: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: cfg.Prefix, + Subsystem: "http", + Name: "requests_total", + Help: "The number of requests handled over time.", + }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + } + + return r +} + +// These methods are called automatically by go-http-metrics/middleware +func (r recorder) ObserveHTTPRequestDuration(_ context.Context, p httpmetrics.HTTPReqProperties, duration time.Duration) { + r.httpRequestDurHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(duration.Seconds()) +} + +func (r recorder) ObserveHTTPResponseSize(_ context.Context, p httpmetrics.HTTPReqProperties, sizeBytes int64) { + r.httpResponseSizeHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(float64(sizeBytes)) +} + +func (r recorder) AddInflightRequests(_ context.Context, p httpmetrics.HTTPProperties, quantity int) { + r.httpRequestsInflight.WithLabelValues(p.Service, p.ID).Add(float64(quantity)) +} + +// New custom method to track all requests made for every REST API request +func (r recorder) AddTotalRequests(_ context.Context, method string, id string) { + r.httpRequestsTotal.WithLabelValues(method, id).Inc() +} diff --git a/module/metrics/unstaked/engine.go b/module/metrics/unstaked/engine.go index d75d6058cd9..e38bb667286 100644 --- a/module/metrics/unstaked/engine.go +++ b/module/metrics/unstaked/engine.go @@ -8,6 +8,8 @@ type EngineCollector struct { metrics module.EngineMetrics } +var _ module.EngineMetrics = (*EngineCollector)(nil) + func NewUnstakedEngineCollector(metrics module.EngineMetrics) *EngineCollector { return &EngineCollector{metrics} } @@ -23,3 +25,11 @@ func (ec *EngineCollector) MessageReceived(engine string, message string) { func (ec *EngineCollector) MessageHandled(engine string, message string) { ec.metrics.MessageHandled("unstaked_"+engine, message) } + +func (ec *EngineCollector) InboundMessageDropped(engine string, message string) { + ec.metrics.InboundMessageDropped("unstaked_"+engine, message) +} + +func (ec *EngineCollector) OutboundMessageDropped(engine string, message string) { + ec.metrics.OutboundMessageDropped("unstaked_"+engine, message) +} diff --git a/module/mock/compliance_metrics.go b/module/mock/compliance_metrics.go index 4c7a91578ca..7ce8c321be8 100644 --- a/module/mock/compliance_metrics.go +++ b/module/mock/compliance_metrics.go @@ -5,8 +5,6 @@ package mock import ( flow "github.com/onflow/flow-go/model/flow" mock "github.com/stretchr/testify/mock" - - time "time" ) // ComplianceMetrics is an autogenerated mock type for the ComplianceMetrics type @@ -19,11 +17,6 @@ func (_m *ComplianceMetrics) BlockFinalized(_a0 *flow.Block) { _m.Called(_a0) } -// BlockProposalDuration provides a mock function with given fields: duration -func (_m *ComplianceMetrics) BlockProposalDuration(duration time.Duration) { - _m.Called(duration) -} - // BlockSealed provides a mock function with given fields: _a0 func (_m *ComplianceMetrics) BlockSealed(_a0 *flow.Block) { _m.Called(_a0) diff --git a/module/mock/engine_metrics.go b/module/mock/engine_metrics.go index 643aaa52e0f..9d10ecb3864 100644 --- a/module/mock/engine_metrics.go +++ b/module/mock/engine_metrics.go @@ -9,6 +9,11 @@ type EngineMetrics struct { mock.Mock } +// InboundMessageDropped provides a mock function with given fields: engine, messages +func (_m *EngineMetrics) InboundMessageDropped(engine string, messages string) { + _m.Called(engine, messages) +} + // MessageHandled provides a mock function with given fields: engine, messages func (_m *EngineMetrics) MessageHandled(engine string, messages string) { _m.Called(engine, messages) @@ -24,6 +29,11 @@ func (_m *EngineMetrics) MessageSent(engine string, message string) { _m.Called(engine, message) } +// OutboundMessageDropped provides a mock function with given fields: engine, messages +func (_m *EngineMetrics) OutboundMessageDropped(engine string, messages string) { + _m.Called(engine, messages) +} + type mockConstructorTestingTNewEngineMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/epoch_lookup.go b/module/mock/epoch_lookup.go index 7c780bd9d06..b3fc9b64e9a 100644 --- a/module/mock/epoch_lookup.go +++ b/module/mock/epoch_lookup.go @@ -9,27 +9,6 @@ type EpochLookup struct { mock.Mock } -// EpochForView provides a mock function with given fields: view -func (_m *EpochLookup) EpochForView(view uint64) (uint64, error) { - ret := _m.Called(view) - - var r0 uint64 - if rf, ok := ret.Get(0).(func(uint64) uint64); ok { - r0 = rf(view) - } else { - r0 = ret.Get(0).(uint64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(uint64) error); ok { - r1 = rf(view) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // EpochForViewWithFallback provides a mock function with given fields: view func (_m *EpochLookup) EpochForViewWithFallback(view uint64) (uint64, error) { ret := _m.Called(view) diff --git a/module/mock/execution_metrics.go b/module/mock/execution_metrics.go index 657f97f16d9..f9731b7fa93 100644 --- a/module/mock/execution_metrics.go +++ b/module/mock/execution_metrics.go @@ -21,6 +21,11 @@ func (_m *ExecutionMetrics) ChunkDataPackRequestProcessed() { _m.Called() } +// ExecutionBlockCachedPrograms provides a mock function with given fields: programs +func (_m *ExecutionMetrics) ExecutionBlockCachedPrograms(programs int) { + _m.Called(programs) +} + // ExecutionBlockDataUploadFinished provides a mock function with given fields: dur func (_m *ExecutionMetrics) ExecutionBlockDataUploadFinished(dur time.Duration) { _m.Called(dur) @@ -181,6 +186,16 @@ func (_m *ExecutionMetrics) RuntimeTransactionParsed(dur time.Duration) { _m.Called(dur) } +// RuntimeTransactionProgramsCacheHit provides a mock function with given fields: +func (_m *ExecutionMetrics) RuntimeTransactionProgramsCacheHit() { + _m.Called() +} + +// RuntimeTransactionProgramsCacheMiss provides a mock function with given fields: +func (_m *ExecutionMetrics) RuntimeTransactionProgramsCacheMiss() { + _m.Called() +} + // StartBlockReceivedToExecuted provides a mock function with given fields: blockID func (_m *ExecutionMetrics) StartBlockReceivedToExecuted(blockID flow.Identifier) { _m.Called(blockID) diff --git a/module/mock/finalizer.go b/module/mock/finalizer.go index 02d4628d8d7..dfe50b916d6 100644 --- a/module/mock/finalizer.go +++ b/module/mock/finalizer.go @@ -26,20 +26,6 @@ func (_m *Finalizer) MakeFinal(blockID flow.Identifier) error { return r0 } -// MakeValid provides a mock function with given fields: blockID -func (_m *Finalizer) MakeValid(blockID flow.Identifier) error { - ret := _m.Called(blockID) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier) error); ok { - r0 = rf(blockID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - type mockConstructorTestingTNewFinalizer interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/hot_stuff.go b/module/mock/hot_stuff.go index 3b5be4c1797..0b0703f6874 100644 --- a/module/mock/hot_stuff.go +++ b/module/mock/hot_stuff.go @@ -3,10 +3,10 @@ package mock import ( - flow "github.com/onflow/flow-go/model/flow" irrecoverable "github.com/onflow/flow-go/module/irrecoverable" - mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // HotStuff is an autogenerated mock type for the HotStuff type @@ -51,20 +51,9 @@ func (_m *HotStuff) Start(_a0 irrecoverable.SignalerContext) { _m.Called(_a0) } -// SubmitProposal provides a mock function with given fields: proposal, parentView -func (_m *HotStuff) SubmitProposal(proposal *flow.Header, parentView uint64) <-chan struct{} { - ret := _m.Called(proposal, parentView) - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func(*flow.Header, uint64) <-chan struct{}); ok { - r0 = rf(proposal, parentView) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 +// SubmitProposal provides a mock function with given fields: proposal +func (_m *HotStuff) SubmitProposal(proposal *model.Proposal) { + _m.Called(proposal) } type mockConstructorTestingTNewHotStuff interface { diff --git a/module/mock/hot_stuff_follower.go b/module/mock/hot_stuff_follower.go index 8e94be63a65..25b5130568e 100644 --- a/module/mock/hot_stuff_follower.go +++ b/module/mock/hot_stuff_follower.go @@ -3,8 +3,10 @@ package mock import ( - flow "github.com/onflow/flow-go/model/flow" + irrecoverable "github.com/onflow/flow-go/module/irrecoverable" mock "github.com/stretchr/testify/mock" + + model "github.com/onflow/flow-go/consensus/hotstuff/model" ) // HotStuffFollower is an autogenerated mock type for the HotStuffFollower type @@ -44,20 +46,14 @@ func (_m *HotStuffFollower) Ready() <-chan struct{} { return r0 } -// SubmitProposal provides a mock function with given fields: proposal, parentView -func (_m *HotStuffFollower) SubmitProposal(proposal *flow.Header, parentView uint64) <-chan struct{} { - ret := _m.Called(proposal, parentView) - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func(*flow.Header, uint64) <-chan struct{}); ok { - r0 = rf(proposal, parentView) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } +// Start provides a mock function with given fields: _a0 +func (_m *HotStuffFollower) Start(_a0 irrecoverable.SignalerContext) { + _m.Called(_a0) +} - return r0 +// SubmitProposal provides a mock function with given fields: proposal +func (_m *HotStuffFollower) SubmitProposal(proposal *model.Proposal) { + _m.Called(proposal) } type mockConstructorTestingTNewHotStuffFollower interface { diff --git a/module/mock/hotstuff_metrics.go b/module/mock/hotstuff_metrics.go index 45d45440dcc..b9c6a18b290 100644 --- a/module/mock/hotstuff_metrics.go +++ b/module/mock/hotstuff_metrics.go @@ -13,6 +13,11 @@ type HotstuffMetrics struct { mock.Mock } +// BlockProcessingDuration provides a mock function with given fields: duration +func (_m *HotstuffMetrics) BlockProcessingDuration(duration time.Duration) { + _m.Called(duration) +} + // CommitteeProcessingDuration provides a mock function with given fields: duration func (_m *HotstuffMetrics) CommitteeProcessingDuration(duration time.Duration) { _m.Called(duration) @@ -58,6 +63,11 @@ func (_m *HotstuffMetrics) SetQCView(view uint64) { _m.Called(view) } +// SetTCView provides a mock function with given fields: view +func (_m *HotstuffMetrics) SetTCView(view uint64) { + _m.Called(view) +} + // SetTimeout provides a mock function with given fields: duration func (_m *HotstuffMetrics) SetTimeout(duration time.Duration) { _m.Called(duration) @@ -68,11 +78,21 @@ func (_m *HotstuffMetrics) SignerProcessingDuration(duration time.Duration) { _m.Called(duration) } +// TimeoutObjectProcessingDuration provides a mock function with given fields: duration +func (_m *HotstuffMetrics) TimeoutObjectProcessingDuration(duration time.Duration) { + _m.Called(duration) +} + // ValidatorProcessingDuration provides a mock function with given fields: duration func (_m *HotstuffMetrics) ValidatorProcessingDuration(duration time.Duration) { _m.Called(duration) } +// VoteProcessingDuration provides a mock function with given fields: duration +func (_m *HotstuffMetrics) VoteProcessingDuration(duration time.Duration) { + _m.Called(duration) +} + type mockConstructorTestingTNewHotstuffMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index 141ffd2398b..06c05bf9d17 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -189,9 +189,9 @@ func (_m *NetworkMetrics) OnPublishedGossipMessagesReceived(count int) { _m.Called(count) } -// OnRateLimitedUnicastMessage provides a mock function with given fields: role, msgType, topic, reason -func (_m *NetworkMetrics) OnRateLimitedUnicastMessage(role string, msgType string, topic string, reason string) { - _m.Called(role, msgType, topic, reason) +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *NetworkMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) } // OnUnauthorizedMessage provides a mock function with given fields: role, msgType, topic, offense diff --git a/module/mock/network_security_metrics.go b/module/mock/network_security_metrics.go index 83f66201521..391bbcdbf31 100644 --- a/module/mock/network_security_metrics.go +++ b/module/mock/network_security_metrics.go @@ -2,16 +2,20 @@ package mock -import mock "github.com/stretchr/testify/mock" +import ( + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) // NetworkSecurityMetrics is an autogenerated mock type for the NetworkSecurityMetrics type type NetworkSecurityMetrics struct { mock.Mock } -// OnRateLimitedUnicastMessage provides a mock function with given fields: role, msgType, topic, reason -func (_m *NetworkSecurityMetrics) OnRateLimitedUnicastMessage(role string, msgType string, topic string, reason string) { - _m.Called(role, msgType, topic, reason) +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *NetworkSecurityMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) } // OnUnauthorizedMessage provides a mock function with given fields: role, msgType, topic, offense diff --git a/module/mock/runtime_metrics.go b/module/mock/runtime_metrics.go index 84aee22d34c..5168f446845 100644 --- a/module/mock/runtime_metrics.go +++ b/module/mock/runtime_metrics.go @@ -33,6 +33,16 @@ func (_m *RuntimeMetrics) RuntimeTransactionParsed(dur time.Duration) { _m.Called(dur) } +// RuntimeTransactionProgramsCacheHit provides a mock function with given fields: +func (_m *RuntimeMetrics) RuntimeTransactionProgramsCacheHit() { + _m.Called() +} + +// RuntimeTransactionProgramsCacheMiss provides a mock function with given fields: +func (_m *RuntimeMetrics) RuntimeTransactionProgramsCacheMiss() { + _m.Called() +} + type mockConstructorTestingTNewRuntimeMetrics interface { mock.TestingT Cleanup(func()) diff --git a/module/signature/signing_tags.go b/module/signature/signing_tags.go index 29fca17c755..9bf00c06cba 100644 --- a/module/signature/signing_tags.go +++ b/module/signature/signing_tags.go @@ -44,6 +44,10 @@ var ( ConsensusVoteTag = tag("Consensus_Vote") // CollectorVoteTag is used for Collection Hotstuff votes CollectorVoteTag = tag("Collector_Vote") + // ConsensusTimeoutTag is used for Consensus Active Pacemaker timeouts + ConsensusTimeoutTag = tag("Consensus_Timeout") + // CollectorTimeoutTag is used for Collector Active Pacemaker timeouts + CollectorTimeoutTag = tag("Collector_Timeout") // ExecutionReceiptTag is used for execution receipts ExecutionReceiptTag = tag("Execution_Receipt") // ResultApprovalTag is used for result approvals diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index b725e5914bb..e2e01cb7929 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -780,7 +780,6 @@ func (m *mockSnapshot) SealedResult() (*flow.ExecutionResult, *flow.Seal, error) func (m *mockSnapshot) Commit() (flow.StateCommitment, error) { return flow.DummyStateCommitment, nil } func (m *mockSnapshot) SealingSegment() (*flow.SealingSegment, error) { return nil, nil } func (m *mockSnapshot) Descendants() ([]flow.Identifier, error) { return nil, nil } -func (m *mockSnapshot) ValidDescendants() ([]flow.Identifier, error) { return nil, nil } func (m *mockSnapshot) RandomSource() ([]byte, error) { return nil, nil } func (m *mockSnapshot) Phase() (flow.EpochPhase, error) { return flow.EpochPhaseUndefined, nil } func (m *mockSnapshot) Epochs() protocol.EpochQuery { return nil } diff --git a/module/trace/constants.go b/module/trace/constants.go index f940d2d4034..90ebc41a706 100644 --- a/module/trace/constants.go +++ b/module/trace/constants.go @@ -103,7 +103,6 @@ const ( EXEGetRegistersWithProofs SpanName = "exe.state.getRegistersWithProofs" EXEGetExecutionResultID SpanName = "exe.state.getExecutionResultID" EXEUpdateHighestExecutedBlockIfHigher SpanName = "exe.state.updateHighestExecutedBlockIfHigher" - EXEHashEvents SpanName = "exe.state.hashEvents" // Verification node // diff --git a/module/util/util.go b/module/util/util.go index 198e173c5ec..1be65b3d9da 100644 --- a/module/util/util.go +++ b/module/util/util.go @@ -3,9 +3,9 @@ package util import ( "context" "reflect" - "sync" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" ) // AllReady calls Ready on all input components and returns a channel that is @@ -40,18 +40,10 @@ func AllClosed(channels ...<-chan struct{}) <-chan struct{} { return done } - var wg sync.WaitGroup - - for _, ch := range channels { - wg.Add(1) - go func(ch <-chan struct{}) { - <-ch - wg.Done() - }(ch) - } - go func() { - wg.Wait() + for _, ch := range channels { + <-ch + } close(done) }() @@ -151,6 +143,37 @@ func WaitError(errChan <-chan error, done <-chan struct{}) error { } } +// readyDoneAwareMerger is a utility structure which implements module.ReadyDoneAware and module.Startable interfaces +// and is used to merge []T into one T. +type readyDoneAwareMerger struct { + components []module.ReadyDoneAware +} + +func (m readyDoneAwareMerger) Start(signalerContext irrecoverable.SignalerContext) { + for _, component := range m.components { + startable, ok := component.(module.Startable) + if ok { + startable.Start(signalerContext) + } + } +} + +func (m readyDoneAwareMerger) Ready() <-chan struct{} { + return AllReady(m.components...) +} + +func (m readyDoneAwareMerger) Done() <-chan struct{} { + return AllDone(m.components...) +} + +var _ module.ReadyDoneAware = (*readyDoneAwareMerger)(nil) +var _ module.Startable = (*readyDoneAwareMerger)(nil) + +// MergeReadyDone merges []module.ReadyDoneAware into one module.ReadyDoneAware. +func MergeReadyDone(components ...module.ReadyDoneAware) module.ReadyDoneAware { + return readyDoneAwareMerger{components: components} +} + // DetypeSlice converts a typed slice containing any kind of elements into an // untyped []any type, in effect removing the element type information from the slice. // It is useful for passing data into structpb.NewValue, which accepts []any but not diff --git a/module/validation/seal_validator.go b/module/validation/seal_validator.go index 0649bf0f7ac..19649abf31f 100644 --- a/module/validation/seal_validator.go +++ b/module/validation/seal_validator.go @@ -178,7 +178,7 @@ func (s *sealValidator) Validate(candidate *flow.Block) (*flow.Seal, error) { } // We do _not_ add the results from the candidate block's own payload to incorporatedResults. - // That's because a result requires to be added to a bock first in order to determine + // That's because a result requires to be added to a block first in order to determine // its chunk assignment for verification. Therefore a seal can only be added in the // next block or after. In other words, a receipt and its seal can't be the same block. diff --git a/network/codec/cbor/codec.go b/network/codec/cbor/codec.go index 4971bc71abb..fa5a6acb451 100644 --- a/network/codec/cbor/codec.go +++ b/network/codec/cbor/codec.go @@ -60,7 +60,7 @@ func (c *Codec) Encode(v interface{}) ([]byte, error) { // encode / append the envelope code //bs1 := binstat.EnterTime(binstat.BinNet + ":wire<1(cbor)envelope2payload") var data bytes.Buffer - data.WriteByte(code) + data.WriteByte(code.Uint8()) //binstat.LeaveVal(bs1, int64(data.Len())) // encode the payload @@ -89,16 +89,17 @@ func (c *Codec) Encode(v interface{}) ([]byte, error) { // - codec.ErrUnknownMsgCode if message code byte does not match any of the configured message codes. // - codec.ErrMsgUnmarshal if the codec fails to unmarshal the data to the message type denoted by the message code. func (c *Codec) Decode(data []byte) (interface{}, error) { - if len(data) == 0 { - return nil, codec.NewInvalidEncodingErr(fmt.Errorf("empty data")) - } + msgCode, err := codec.MessageCodeFromPayload(data) + if err != nil { + return nil, err + } // decode the envelope //bs1 := binstat.EnterTime(binstat.BinNet + ":wire>3(cbor)payload2envelope") //binstat.LeaveVal(bs1, int64(len(data))) - msgInterface, what, err := codec.InterfaceFromMessageCode(data[0]) + msgInterface, what, err := codec.InterfaceFromMessageCode(msgCode) if err != nil { return nil, err } diff --git a/network/codec/cbor/codec_test.go b/network/codec/cbor/codec_test.go index ad32b0c89c7..8d3c555b87d 100644 --- a/network/codec/cbor/codec_test.go +++ b/network/codec/cbor/codec_test.go @@ -43,19 +43,19 @@ func TestCodec_Decode(t *testing.T) { t.Run("returns error when message code is invalid", func(t *testing.T) { t.Parallel() - decoded, err := c.Decode([]byte{codec.CodeMin}) + decoded, err := c.Decode([]byte{codec.CodeMin.Uint8()}) assert.Nil(t, decoded) assert.True(t, codec.IsErrUnknownMsgCode(err)) - decoded, err = c.Decode([]byte{codec.CodeMax}) + decoded, err = c.Decode([]byte{codec.CodeMax.Uint8()}) assert.Nil(t, decoded) assert.True(t, codec.IsErrUnknownMsgCode(err)) - decoded, err = c.Decode([]byte{codec.CodeMin - 1}) + decoded, err = c.Decode([]byte{codec.CodeMin.Uint8() - 1}) assert.Nil(t, decoded) assert.True(t, codec.IsErrUnknownMsgCode(err)) - decoded, err = c.Decode([]byte{codec.CodeMax + 1}) + decoded, err = c.Decode([]byte{codec.CodeMax.Uint8() + 1}) assert.Nil(t, decoded) assert.True(t, codec.IsErrUnknownMsgCode(err)) }) @@ -63,7 +63,7 @@ func TestCodec_Decode(t *testing.T) { t.Run("returns error when unmarshalling fails - empty", func(t *testing.T) { t.Parallel() - decoded, err := c.Decode([]byte{codec.CodeBlockProposal}) + decoded, err := c.Decode([]byte{codec.CodeBlockProposal.Uint8()}) assert.Nil(t, decoded) assert.True(t, codec.IsErrMsgUnmarshal(err)) }) @@ -75,7 +75,7 @@ func TestCodec_Decode(t *testing.T) { encoded, err := c.Encode(data) require.NoError(t, err) - encoded[0] = codec.CodeCollectionGuarantee + encoded[0] = codec.CodeCollectionGuarantee.Uint8() decoded, err := c.Decode(encoded) assert.Nil(t, decoded) diff --git a/network/codec/cbor/decoder.go b/network/codec/cbor/decoder.go index 9486c6000e1..77dd48bd82c 100644 --- a/network/codec/cbor/decoder.go +++ b/network/codec/cbor/decoder.go @@ -3,8 +3,6 @@ package cbor import ( - "fmt" - "github.com/fxamacker/cbor/v2" "github.com/onflow/flow-go/network/codec" @@ -32,11 +30,12 @@ func (d *Decoder) Decode() (interface{}, error) { return nil, codec.NewInvalidEncodingErr(err) } - if len(data) == 0 { - return nil, codec.NewInvalidEncodingErr(fmt.Errorf("empty data")) + msgCode, err := codec.MessageCodeFromPayload(data) + if err != nil { + return nil, err } - msgInterface, what, err := codec.InterfaceFromMessageCode(data[0]) + msgInterface, what, err := codec.InterfaceFromMessageCode(msgCode) if err != nil { return nil, err } diff --git a/network/codec/cbor/decoder_test.go b/network/codec/cbor/decoder_test.go index 21bea8bf705..1f71cb5e306 100644 --- a/network/codec/cbor/decoder_test.go +++ b/network/codec/cbor/decoder_test.go @@ -94,10 +94,10 @@ func TestDecoder_Decode(t *testing.T) { // in this test, only the first byte is set since there should be an error after reading // the invalid code datas := [][]byte{ - {codec.CodeMin - 1}, // code < min - {codec.CodeMin}, // code == min - {codec.CodeMax}, // code == max - {codec.CodeMax + 1}, // code > max + {codec.CodeMin.Uint8() - 1}, // code < min + {codec.CodeMin.Uint8()}, // code == min + {codec.CodeMax.Uint8()}, // code == max + {codec.CodeMax.Uint8() + 1}, // code > max } for i := range datas { @@ -117,7 +117,7 @@ func TestDecoder_Decode(t *testing.T) { var buf bytes.Buffer - err := cborcodec.NewCodec().NewEncoder(&buf).Encode([]byte{codec.CodeBlockProposal}) + err := cborcodec.NewCodec().NewEncoder(&buf).Encode([]byte{codec.CodeBlockProposal.Uint8()}) require.NoError(t, err) decoded, err := c.NewDecoder(&buf).Decode() @@ -130,7 +130,7 @@ func TestDecoder_Decode(t *testing.T) { // first encode the message to bytes with an incorrect type var data bytes.Buffer - _ = data.WriteByte(codec.CodeCollectionGuarantee) + _ = data.WriteByte(codec.CodeCollectionGuarantee.Uint8()) encoder := cborcodec.EncMode.NewEncoder(&data) err := encoder.Encode(blockProposal) require.NoError(t, err) @@ -151,7 +151,7 @@ func TestDecoder_Decode(t *testing.T) { // first encode the message to bytes var data bytes.Buffer - _ = data.WriteByte(codec.CodeBlockProposal) + _ = data.WriteByte(codec.CodeBlockProposal.Uint8()) encoder := cborcodec.EncMode.NewEncoder(&data) err := encoder.Encode(blockProposal) require.NoError(t, err) diff --git a/network/codec/cbor/encoder.go b/network/codec/cbor/encoder.go index 5ca1cd11cbb..c154ddebead 100644 --- a/network/codec/cbor/encoder.go +++ b/network/codec/cbor/encoder.go @@ -30,7 +30,7 @@ func (e *Encoder) Encode(v interface{}) error { // encode the payload //bs1 := binstat.EnterTime(fmt.Sprintf("%s%s%s:%d", binstat.BinNet, ":strm<1(cbor)", what, code)) // e.g. ~3net::strm<1(cbor)CodeEntityRequest:23 var data bytes.Buffer - _ = data.WriteByte(code) + _ = data.WriteByte(code.Uint8()) encoder := cborcodec.EncMode.NewEncoder(&data) err = encoder.Encode(v) //binstat.LeaveVal(bs1, int64(data.Len())) diff --git a/network/codec/codes.go b/network/codec/codes.go index 91006ab1d08..629219fed3b 100644 --- a/network/codec/codes.go +++ b/network/codec/codes.go @@ -10,12 +10,19 @@ import ( "github.com/onflow/flow-go/model/messages" ) +type MessageCode uint8 + +func (m MessageCode) Uint8() uint8 { + return uint8(m) +} + const ( - CodeMin uint8 = iota + 1 + CodeMin MessageCode = iota + 1 // consensus CodeBlockProposal CodeBlockVote + CodeTimeoutObject // protocol state sync CodeSyncRequest @@ -28,6 +35,7 @@ const ( CodeClusterBlockProposal CodeClusterBlockVote CodeClusterBlockResponse + CodeClusterTimeoutObject // collections, guarantees & transactions CodeCollectionGuarantee @@ -64,79 +72,84 @@ const ( ) // MessageCodeFromInterface returns the correct Code based on the underlying type of message v. -func MessageCodeFromInterface(v interface{}) (uint8, string, error) { +func MessageCodeFromInterface(v interface{}) (MessageCode, string, error) { + s := what(v) switch v.(type) { // consensus case *messages.BlockProposal: - return CodeBlockProposal, "CodeBlockProposal", nil + return CodeBlockProposal, s, nil case *messages.BlockVote: - return CodeBlockVote, "CodeBlockVote", nil + return CodeBlockVote, s, nil + case *messages.TimeoutObject: + return CodeTimeoutObject, s, nil // cluster consensus case *messages.ClusterBlockProposal: - return CodeClusterBlockProposal, "CodeClusterBlockProposal", nil + return CodeClusterBlockProposal, s, nil case *messages.ClusterBlockVote: - return CodeClusterBlockVote, "CodeClusterBlockVote", nil + return CodeClusterBlockVote, s, nil case *messages.ClusterBlockResponse: - return CodeClusterBlockResponse, "CodeClusterBlockResponse", nil + return CodeClusterBlockResponse, s, nil + case *messages.ClusterTimeoutObject: + return CodeClusterTimeoutObject, s, nil // protocol state sync case *messages.SyncRequest: - return CodeSyncRequest, "CodeSyncRequest", nil + return CodeSyncRequest, s, nil case *messages.SyncResponse: - return CodeSyncResponse, "CodeSyncResponse", nil + return CodeSyncResponse, s, nil case *messages.RangeRequest: - return CodeRangeRequest, "CodeRangeRequest", nil + return CodeRangeRequest, s, nil case *messages.BatchRequest: - return CodeBatchRequest, "CodeBatchRequest", nil + return CodeBatchRequest, s, nil case *messages.BlockResponse: - return CodeBlockResponse, "CodeBlockResponse", nil + return CodeBlockResponse, s, nil // collections, guarantees & transactions case *flow.CollectionGuarantee: - return CodeCollectionGuarantee, "CodeCollectionGuarantee", nil + return CodeCollectionGuarantee, s, nil case *flow.TransactionBody: - return CodeTransactionBody, "CodeTransactionBody", nil + return CodeTransactionBody, s, nil case *flow.Transaction: - return CodeTransaction, "CodeTransaction", nil + return CodeTransaction, s, nil // core messages for execution & verification case *flow.ExecutionReceipt: - return CodeExecutionReceipt, "CodeExecutionReceipt", nil + return CodeExecutionReceipt, s, nil case *flow.ResultApproval: - return CodeResultApproval, "CodeResultApproval", nil + return CodeResultApproval, s, nil // execution state synchronization case *messages.ExecutionStateSyncRequest: - return CodeExecutionStateSyncRequest, "CodeExecutionStateSyncRequest", nil + return CodeExecutionStateSyncRequest, s, nil case *messages.ExecutionStateDelta: - return CodeExecutionStateDelta, "CodeExecutionStateDelta", nil + return CodeExecutionStateDelta, s, nil // data exchange for execution of blocks case *messages.ChunkDataRequest: - return CodeChunkDataRequest, "CodeChunkDataRequest", nil + return CodeChunkDataRequest, s, nil case *messages.ChunkDataResponse: - return CodeChunkDataResponse, "CodeChunkDataResponse", nil + return CodeChunkDataResponse, s, nil // result approvals case *messages.ApprovalRequest: - return CodeApprovalRequest, "CodeApprovalRequest", nil + return CodeApprovalRequest, s, nil case *messages.ApprovalResponse: - return CodeApprovalResponse, "CodeApprovalResponse", nil + return CodeApprovalResponse, s, nil // generic entity exchange engines case *messages.EntityRequest: - return CodeEntityRequest, "CodeEntityRequest", nil + return CodeEntityRequest, s, nil case *messages.EntityResponse: - return CodeEntityResponse, "CodeEntityResponse", nil + return CodeEntityResponse, s, nil // testing case *message.TestMessage: - return CodeEcho, "CodeEcho", nil + return CodeEcho, s, nil // dkg case *messages.DKGMessage: - return CodeDKGMessage, "CodeDKGMessage", nil + return CodeDKGMessage, s, nil default: return 0, "", fmt.Errorf("invalid encode type (%T)", v) @@ -147,81 +160,100 @@ func MessageCodeFromInterface(v interface{}) (uint8, string, error) { // of the message code represents. // Expected error returns during normal operations: // - ErrUnknownMsgCode if message code does not match any of the configured message codes above. -func InterfaceFromMessageCode(code uint8) (interface{}, string, error) { +func InterfaceFromMessageCode(code MessageCode) (interface{}, string, error) { switch code { // consensus case CodeBlockProposal: - return &messages.BlockProposal{}, "BlockProposal", nil + return &messages.BlockProposal{}, what(&messages.BlockProposal{}), nil case CodeBlockVote: - return &messages.BlockVote{}, "BlockVote", nil + return &messages.BlockVote{}, what(&messages.BlockVote{}), nil + case CodeTimeoutObject: + return &messages.TimeoutObject{}, what(&messages.TimeoutObject{}), nil // cluster consensus case CodeClusterBlockProposal: - return &messages.ClusterBlockProposal{}, "ClusterBlockProposal", nil + return &messages.ClusterBlockProposal{}, what(&messages.ClusterBlockProposal{}), nil case CodeClusterBlockVote: - return &messages.ClusterBlockVote{}, "ClusterBlockVote", nil + return &messages.ClusterBlockVote{}, what(&messages.ClusterBlockVote{}), nil case CodeClusterBlockResponse: - return &messages.ClusterBlockResponse{}, "ClusterBlockResponse", nil + return &messages.ClusterBlockResponse{}, what(&messages.ClusterBlockResponse{}), nil + case CodeClusterTimeoutObject: + return &messages.ClusterTimeoutObject{}, what(&messages.ClusterTimeoutObject{}), nil // protocol state sync case CodeSyncRequest: - return &messages.SyncRequest{}, "SyncRequest", nil + return &messages.SyncRequest{}, what(&messages.SyncRequest{}), nil case CodeSyncResponse: - return &messages.SyncResponse{}, "SyncResponse", nil + return &messages.SyncResponse{}, what(&messages.SyncResponse{}), nil case CodeRangeRequest: - return &messages.RangeRequest{}, "RangeRequest", nil + return &messages.RangeRequest{}, what(&messages.RangeRequest{}), nil case CodeBatchRequest: - return &messages.BatchRequest{}, "BatchRequest", nil + return &messages.BatchRequest{}, what(&messages.BatchRequest{}), nil case CodeBlockResponse: - return &messages.BlockResponse{}, "BlockResponse", nil + return &messages.BlockResponse{}, what(&messages.BlockResponse{}), nil // collections, guarantees & transactions case CodeCollectionGuarantee: - return &flow.CollectionGuarantee{}, "CollectionGuarantee", nil + return &flow.CollectionGuarantee{}, what(&flow.CollectionGuarantee{}), nil case CodeTransactionBody: - return &flow.TransactionBody{}, "TransactionBody", nil + return &flow.TransactionBody{}, what(&flow.TransactionBody{}), nil case CodeTransaction: - return &flow.Transaction{}, "Transaction", nil + return &flow.Transaction{}, what(&flow.Transaction{}), nil // core messages for execution & verification case CodeExecutionReceipt: - return &flow.ExecutionReceipt{}, "ExecutionReceipt", nil + return &flow.ExecutionReceipt{}, what(&flow.ExecutionReceipt{}), nil case CodeResultApproval: - return &flow.ResultApproval{}, "ResultApproval", nil + return &flow.ResultApproval{}, what(&flow.ResultApproval{}), nil // execution state synchronization case CodeExecutionStateSyncRequest: - return &messages.ExecutionStateSyncRequest{}, "ExecutionStateSyncRequest", nil + return &messages.ExecutionStateSyncRequest{}, what(&messages.ExecutionStateSyncRequest{}), nil case CodeExecutionStateDelta: - return &messages.ExecutionStateDelta{}, "ExecutionStateDelta", nil + return &messages.ExecutionStateDelta{}, what(&messages.ExecutionStateDelta{}), nil // data exchange for execution of blocks case CodeChunkDataRequest: - return &messages.ChunkDataRequest{}, "ChunkDataRequest", nil + return &messages.ChunkDataRequest{}, what(&messages.ChunkDataRequest{}), nil case CodeChunkDataResponse: - return &messages.ChunkDataResponse{}, "ChunkDataResponse", nil + return &messages.ChunkDataResponse{}, what(&messages.ChunkDataResponse{}), nil // result approvals case CodeApprovalRequest: - return &messages.ApprovalRequest{}, "ApprovalRequest", nil + return &messages.ApprovalRequest{}, what(&messages.ApprovalRequest{}), nil case CodeApprovalResponse: - return &messages.ApprovalResponse{}, "ApprovalResponse", nil + return &messages.ApprovalResponse{}, what(&messages.ApprovalResponse{}), nil // generic entity exchange engines case CodeEntityRequest: - return &messages.EntityRequest{}, "EntityRequest", nil + return &messages.EntityRequest{}, what(&messages.EntityRequest{}), nil case CodeEntityResponse: - return &messages.EntityResponse{}, "EntityResponse", nil + return &messages.EntityResponse{}, what(&messages.EntityResponse{}), nil // dkg case CodeDKGMessage: - return &messages.DKGMessage{}, "DKGMessage", nil + return &messages.DKGMessage{}, what(&messages.DKGMessage{}), nil // test messages case CodeEcho: - return &message.TestMessage{}, "TestMessage", nil + return &message.TestMessage{}, what(&message.TestMessage{}), nil default: return nil, "", NewUnknownMsgCodeErr(code) } } + +// MessageCodeFromPayload checks the length of the payload bytes before returning the first byte encoded MessageCode. +// Expected error returns during normal operations: +// - ErrInvalidEncoding if payload is empty +func MessageCodeFromPayload(payload []byte) (MessageCode, error) { + if len(payload) == 0 { + return 0, NewInvalidEncodingErr(fmt.Errorf("empty payload")) + } + + return MessageCode(payload[0]), nil +} + +func what(v interface{}) string { + return fmt.Sprintf("%T", v) +} diff --git a/network/codec/errors.go b/network/codec/errors.go index 2b9d7c8dfeb..752a702762e 100644 --- a/network/codec/errors.go +++ b/network/codec/errors.go @@ -27,7 +27,7 @@ func IsErrInvalidEncoding(err error) bool { // ErrUnknownMsgCode indicates that the message code byte (first byte of message payload) is unknown. type ErrUnknownMsgCode struct { - code uint8 + code MessageCode } func (e ErrUnknownMsgCode) Error() string { @@ -35,7 +35,7 @@ func (e ErrUnknownMsgCode) Error() string { } // NewUnknownMsgCodeErr returns a new ErrUnknownMsgCode -func NewUnknownMsgCodeErr(code uint8) ErrUnknownMsgCode { +func NewUnknownMsgCodeErr(code MessageCode) ErrUnknownMsgCode { return ErrUnknownMsgCode{code} } diff --git a/network/codec/roundTripHeader_test.go b/network/codec/roundTripHeader_test.go index 08effb02a18..382b7fd808b 100644 --- a/network/codec/roundTripHeader_test.go +++ b/network/codec/roundTripHeader_test.go @@ -1,7 +1,6 @@ package codec_test import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -30,10 +29,16 @@ func roundTripHeaderViaCodec(t *testing.T, codec network.Codec) { assert.NoError(t, err) decoded := decodedInterface.(*messages.BlockProposal) decodedBlock := decoded.Block.ToInternal() - assert.Equal(t, block.Header.ProposerSigData, decodedBlock.Header.ProposerSigData) - messageHeader := fmt.Sprintf("- .Header=%+v\n", block.Header) - decodedHeader := fmt.Sprintf("- .Header=%+v\n", decodedBlock.Header) - assert.Equal(t, messageHeader, decodedHeader) + // compare LastViewTC separately, because it is a pointer field + if decodedBlock.Header.LastViewTC == nil { + assert.Equal(t, block.Header.LastViewTC, decodedBlock.Header.LastViewTC) + } else { + assert.Equal(t, *block.Header.LastViewTC, *decodedBlock.Header.LastViewTC) + } + // compare the rest of the header + // manually set LastViewTC fields to be equal to pass the Header pointer comparison + decodedBlock.Header.LastViewTC = block.Header.LastViewTC + assert.Equal(t, *block.Header, *decodedBlock.Header) } func TestRoundTripHeaderViaCBOR(t *testing.T) { diff --git a/network/errors.go b/network/errors.go index 07ecbc23aa1..7165c05fd1f 100644 --- a/network/errors.go +++ b/network/errors.go @@ -2,8 +2,39 @@ package network import ( "errors" + "fmt" ) var ( EmptyTargetList = errors.New("target list empty") ) + +// TransientError represents an error returned from a network layer function call +// which may be interpreted as non-critical. In general, we desire that all expected +// error return values are enumerated in a function's documentation - any undocumented +// errors are considered fatal. However, 3rd party libraries don't always conform to +// this standard, including the networking libraries we use. This error type can be +// used to wrap these 3rd party errors on the boundary into flow-go, to explicitly +// mark them as non-critical. +type TransientError struct { + Err error +} + +func (err TransientError) Error() string { + return err.Err.Error() +} + +func (err TransientError) Unwrap() error { + return err.Err +} + +func NewTransientErrorf(msg string, args ...interface{}) TransientError { + return TransientError{ + Err: fmt.Errorf(msg, args...), + } +} + +func IsTransientError(err error) bool { + var errTransient TransientError + return errors.As(err, &errTransient) +} diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index 7ed95a51295..f71cb3ab011 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/libp2p/go-libp2p/core/network" + "github.com/onflow/flow-go/network/message" addrutil "github.com/libp2p/go-addr-util" @@ -25,7 +27,6 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" - flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2putils" "github.com/onflow/flow-go/network/internal/testutils" @@ -212,12 +213,19 @@ func AddNodesToEachOthersPeerStore(t *testing.T, nodes []p2p.LibP2PNode, ids flo // EnsureNotConnected ensures that no connection exists from "from" nodes to "to" nodes. func EnsureNotConnected(t *testing.T, ctx context.Context, from []p2p.LibP2PNode, to []p2p.LibP2PNode) { - for _, node := range from { + for _, this := range from { for _, other := range to { - if node == other { + if this == other { require.Fail(t, "overlapping nodes in from and to lists") } - require.Error(t, node.Host().Connect(ctx, other.Host().Peerstore().PeerInfo(other.Host().ID()))) + thisId := this.Host().ID() + // we intentionally do not check the error here, with libp2p v0.24 connection gating at the "InterceptSecured" level + // does not cause the nodes to complain about the connection being rejected at the dialer side. + // Hence, we instead check for any trace of the connection being established in the receiver side. + _ = this.Host().Connect(ctx, other.Host().Peerstore().PeerInfo(other.Host().ID())) + // ensures that other node has never received a connection from this node. + require.Equal(t, other.Host().Network().Connectedness(thisId), network.NotConnected) + require.Empty(t, other.Host().Network().ConnsToPeer(thisId)) } } } @@ -235,11 +243,8 @@ func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p _, topic := messageFactory() subs := make([]p2p.Subscription, len(to)) - svc := unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector()) tv := validator.TopicValidator( unittest.Logger(), - unittest.NetworkCodec(), - svc, unittest.AllowAllPeerFilter()) var err error for _, node := range from { @@ -334,10 +339,24 @@ func EnsureNoStreamCreation(t *testing.T, ctx context.Context, from []p2p.LibP2P // should not happen, unless the test is misconfigured. require.Fail(t, "node is in both from and to lists") } - // stream creation should fail - _, err := this.CreateStream(ctx, other.Host().ID()) - require.Error(t, err) - require.True(t, flownet.IsPeerUnreachableError(err)) + + // we intentionally do not check the error here, with libp2p v0.24 connection gating at the "InterceptSecured" level + // does not cause the nodes to complain about the connection being rejected at the dialer side. + // Hence, we instead check for any trace of the connection being established in the receiver side. + otherId := other.Host().ID() + thisId := this.Host().ID() + + // closes all connections from other node to this node in order to isolate the connection attempt. + for _, conn := range other.Host().Network().ConnsToPeer(thisId) { + require.NoError(t, conn.Close()) + } + require.Empty(t, other.Host().Network().ConnsToPeer(thisId)) + + _, err := this.CreateStream(ctx, otherId) + // ensures that other node has never received a connection from this node. + require.Equal(t, other.Host().Network().Connectedness(thisId), network.NotConnected) + // a stream is established on top of a connection, so if there is no connection, there should be no stream. + require.Empty(t, other.Host().Network().ConnsToPeer(thisId)) // runs the error checkers if any. for _, check := range errorCheckers { diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index 32476d0511f..a2eb87bb0c8 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -2,6 +2,7 @@ package testutils import ( "context" + "fmt" "reflect" "runtime" "strings" @@ -48,56 +49,76 @@ import ( var sporkID = unittest.IdentifierFixture() +// RateLimitConsumer p2p.RateLimiterConsumer fixture that invokes a callback when rate limit event is consumed. +type RateLimitConsumer struct { + callback func(pid peer.ID, role, msgType, topic, reason string) // callback func that will be invoked on rate limit +} + +func (r *RateLimitConsumer) OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) { + r.callback(pid, role, msgType, topic, reason) +} + type PeerTag struct { Peer peer.ID Tag string } +// TagWatchingConnManager implements connection.ConnManager struct, and manages connections with tags. It +// also maintains a set of observers that it notifies when a tag is added or removed from a peer. type TagWatchingConnManager struct { *connection.ConnManager observers map[observable.Observer]struct{} obsLock sync.RWMutex } -func (cwcm *TagWatchingConnManager) Subscribe(observer observable.Observer) { - cwcm.obsLock.Lock() - defer cwcm.obsLock.Unlock() +// Subscribe allows an observer to subscribe to receive notifications when a tag is added or removed from a peer. +func (tw *TagWatchingConnManager) Subscribe(observer observable.Observer) { + tw.obsLock.Lock() + defer tw.obsLock.Unlock() var void struct{} - cwcm.observers[observer] = void + tw.observers[observer] = void } -func (cwcm *TagWatchingConnManager) Unsubscribe(observer observable.Observer) { - cwcm.obsLock.Lock() - defer cwcm.obsLock.Unlock() - delete(cwcm.observers, observer) +// Unsubscribe allows an observer to unsubscribe from receiving notifications. +func (tw *TagWatchingConnManager) Unsubscribe(observer observable.Observer) { + tw.obsLock.Lock() + defer tw.obsLock.Unlock() + delete(tw.observers, observer) } -func (cwcm *TagWatchingConnManager) Protect(id peer.ID, tag string) { - cwcm.obsLock.RLock() - defer cwcm.obsLock.RUnlock() - cwcm.ConnManager.Protect(id, tag) - for obs := range cwcm.observers { +// Protect adds a tag to a peer. It also notifies all observers that a tag has been added to a peer. +func (tw *TagWatchingConnManager) Protect(id peer.ID, tag string) { + tw.obsLock.RLock() + defer tw.obsLock.RUnlock() + tw.ConnManager.Protect(id, tag) + for obs := range tw.observers { go obs.OnNext(PeerTag{Peer: id, Tag: tag}) } } -func (cwcm *TagWatchingConnManager) Unprotect(id peer.ID, tag string) bool { - cwcm.obsLock.RLock() - defer cwcm.obsLock.RUnlock() - res := cwcm.ConnManager.Unprotect(id, tag) - for obs := range cwcm.observers { +// Unprotect removes a tag from a peer. It also notifies all observers that a tag has been removed from a peer. +func (tw *TagWatchingConnManager) Unprotect(id peer.ID, tag string) bool { + tw.obsLock.RLock() + defer tw.obsLock.RUnlock() + res := tw.ConnManager.Unprotect(id, tag) + for obs := range tw.observers { go obs.OnNext(PeerTag{Peer: id, Tag: tag}) } return res } -func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnectionMetrics) *TagWatchingConnManager { - cm := connection.NewConnManager(log, metrics) +// NewTagWatchingConnManager creates a new TagWatchingConnManager with the given config. It returns an error if the config is invalid. +func NewTagWatchingConnManager(log zerolog.Logger, metrics module.LibP2PConnectionMetrics, config *connection.ManagerConfig) (*TagWatchingConnManager, error) { + cm, err := connection.NewConnManager(log, metrics, config) + if err != nil { + return nil, fmt.Errorf("could not create connection manager: %w", err) + } + return &TagWatchingConnManager{ ConnManager: cm, observers: make(map[observable.Observer]struct{}), obsLock: sync.RWMutex{}, - } + }, nil } // GenerateIDs is a test helper that generate flow identities with a valid port and libp2p nodes. @@ -107,33 +128,39 @@ func GenerateIDs(t *testing.T, logger zerolog.Logger, n int, opts ...func(*optsC libP2PNodes := make([]p2p.LibP2PNode, n) tagObservables := make([]observable.Observable, n) - o := &optsConfig{peerUpdateInterval: connection.DefaultPeerUpdateInterval} + identities := unittest.IdentityListFixture(n, unittest.WithAllRoles()) + idProvider := NewUpdatableIDProvider(identities) + o := &optsConfig{ + peerUpdateInterval: connection.DefaultPeerUpdateInterval, + unicastRateLimiterDistributor: ratelimit.NewUnicastRateLimiterDistributor(), + connectionGater: NewConnectionGater(idProvider, func(p peer.ID) error { + return nil + }), + } for _, opt := range opts { opt(o) } - identities := unittest.IdentityListFixture(n, unittest.WithAllRoles()) - for _, identity := range identities { for _, idOpt := range o.idOpts { idOpt(identity) } } - idProvider := id.NewFixedIdentityProvider(identities) - // generates keys and address for the node - for i, id := range identities { + for i, identity := range identities { // generate key - key, err := generateNetworkingKey(id.NodeID) + key, err := generateNetworkingKey(identity.NodeID) require.NoError(t, err) var opts []nodeBuilderOption opts = append(opts, withDHT(o.dhtPrefix, o.dhtOpts...)) opts = append(opts, withPeerManagerOptions(connection.ConnectionPruningEnabled, o.peerUpdateInterval)) + opts = append(opts, withRateLimiterDistributor(o.unicastRateLimiterDistributor)) + opts = append(opts, withConnectionGater(o.connectionGater)) - libP2PNodes[i], tagObservables[i] = generateLibP2PNode(t, logger, key, idProvider, opts...) + libP2PNodes[i], tagObservables[i] = generateLibP2PNode(t, logger, key, opts...) _, port, err := libP2PNodes[i].GetIPPort() require.NoError(t, err) @@ -160,6 +187,7 @@ func GenerateMiddlewares(t *testing.T, peerUpdateInterval: connection.DefaultPeerUpdateInterval, unicastRateLimiters: ratelimit.NoopRateLimiters(), networkMetrics: metrics.NewNoopCollector(), + peerManagerFilters: []p2p.PeerFilter{}, } for _, opt := range opts { @@ -185,7 +213,8 @@ func GenerateMiddlewares(t *testing.T, translator.NewIdentityProviderIDTranslator(idProviders[i]), codec, consumer, - middleware.WithUnicastRateLimiters(o.unicastRateLimiters)) + middleware.WithUnicastRateLimiters(o.unicastRateLimiters), + middleware.WithPeerManagerFilters(o.peerManagerFilters)) } return mws, idProviders } @@ -243,12 +272,21 @@ func GenerateIDsAndMiddlewares(t *testing.T, } type optsConfig struct { - idOpts []func(*flow.Identity) - dhtPrefix string - dhtOpts []dht.Option - unicastRateLimiters *ratelimit.RateLimiters - peerUpdateInterval time.Duration - networkMetrics module.NetworkMetrics + idOpts []func(*flow.Identity) + dhtPrefix string + dhtOpts []dht.Option + unicastRateLimiters *ratelimit.RateLimiters + peerUpdateInterval time.Duration + networkMetrics module.NetworkMetrics + peerManagerFilters []p2p.PeerFilter + unicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor + connectionGater connmgr.ConnectionGater +} + +func WithUnicastRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) func(*optsConfig) { + return func(o *optsConfig) { + o.unicastRateLimiterDistributor = distributor + } } func WithIdentityOpts(idOpts ...func(*flow.Identity)) func(*optsConfig) { @@ -270,12 +308,24 @@ func WithPeerUpdateInterval(interval time.Duration) func(*optsConfig) { } } +func WithPeerManagerFilters(filters ...p2p.PeerFilter) func(*optsConfig) { + return func(o *optsConfig) { + o.peerManagerFilters = filters + } +} + func WithUnicastRateLimiters(limiters *ratelimit.RateLimiters) func(*optsConfig) { return func(o *optsConfig) { o.unicastRateLimiters = limiters } } +func WithConnectionGater(connectionGater connmgr.ConnectionGater) func(*optsConfig) { + return func(o *optsConfig) { + o.connectionGater = connectionGater + } +} + func WithNetworkMetrics(m module.NetworkMetrics) func(*optsConfig) { return func(o *optsConfig) { o.networkMetrics = m @@ -357,17 +407,29 @@ func withPeerManagerOptions(connectionPruning bool, updateInterval time.Duration } } +func withRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) nodeBuilderOption { + return func(nb p2pbuilder.NodeBuilder) { + nb.SetRateLimiterDistributor(distributor) + } +} + +func withConnectionGater(connectionGater connmgr.ConnectionGater) nodeBuilderOption { + return func(nb p2pbuilder.NodeBuilder) { + nb.SetConnectionGater(connectionGater) + } +} + // generateLibP2PNode generates a `LibP2PNode` on localhost using a port assigned by the OS func generateLibP2PNode(t *testing.T, logger zerolog.Logger, key crypto.PrivateKey, - idProvider module.IdentityProvider, opts ...nodeBuilderOption) (p2p.LibP2PNode, observable.Observable) { noopMetrics := metrics.NewNoopCollector() // Inject some logic to be able to observe connections of this node - connManager := NewTagWatchingConnManager(logger, noopMetrics) + connManager, err := NewTagWatchingConnManager(logger, noopMetrics, connection.DefaultConnManagerConfig()) + require.NoError(t, err) builder := p2pbuilder.NewNodeBuilder( logger, @@ -454,13 +516,31 @@ func NetworkPayloadFixture(t *testing.T, size uint) []byte { // NewResourceManager creates a new resource manager for testing with no limits. func NewResourceManager(t *testing.T) p2pNetwork.ResourceManager { - return p2pNetwork.NullResourceManager + return &p2pNetwork.NullResourceManager{} } // NewConnectionGater creates a new connection gater for testing with given allow listing filter. -func NewConnectionGater(allowListFilter p2p.PeerFilter) connmgr.ConnectionGater { +func NewConnectionGater(idProvider module.IdentityProvider, allowListFilter p2p.PeerFilter) connmgr.ConnectionGater { filters := []p2p.PeerFilter{allowListFilter} return connection.NewConnGater(unittest.Logger(), + idProvider, connection.WithOnInterceptPeerDialFilters(filters), connection.WithOnInterceptSecuredFilters(filters)) + +} + +// IsRateLimitedPeerFilter returns a p2p.PeerFilter that will return an error if the peer is rate limited. +func IsRateLimitedPeerFilter(rateLimiter p2p.RateLimiter) p2p.PeerFilter { + return func(p peer.ID) error { + if rateLimiter.IsRateLimited(p) { + return fmt.Errorf("peer is rate limited") + } + + return nil + } +} + +// NewRateLimiterConsumer returns a p2p.RateLimiterConsumer fixture that will invoke the callback provided. +func NewRateLimiterConsumer(callback func(pid peer.ID, role, msgType, topic, reason string)) p2p.RateLimiterConsumer { + return &RateLimitConsumer{callback} } diff --git a/network/message/authorization.go b/network/message/authorization.go index 570e1ccb091..9b16a90141f 100644 --- a/network/message/authorization.go +++ b/network/message/authorization.go @@ -37,7 +37,7 @@ type MsgAuthConfig struct { // - ErrUnauthorizedRole: the role is not included in the message's list of authorized roles for the provided channel // - ErrUnauthorizedUnicastOnChannel: the message is not authorized to be sent via unicast protocol. // - ErrUnauthorizedPublishOnChannel: the message is not authorized to be sent via publish protocol. -func (m MsgAuthConfig) EnsureAuthorized(role flow.Role, channel channels.Channel, protocol Protocol) error { +func (m MsgAuthConfig) EnsureAuthorized(role flow.Role, channel channels.Channel, protocol ProtocolType) error { authConfig, ok := m.Config[channel] if !ok { return ErrUnauthorizedMessageOnChannel @@ -66,11 +66,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ConsensusCommittee: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, channels.PushBlocks: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ReceiveBlocks = PushBlocks }, } @@ -82,7 +82,19 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ConsensusCommittee: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, + }, + }, + } + authorizationConfigs[TimeoutObject] = MsgAuthConfig{ + Name: TimeoutObject, + Type: func() interface{} { + return new(messages.TimeoutObject) + }, + Config: map[channels.Channel]ChannelAuthConfig{ + channels.ConsensusCommittee: { + AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -96,11 +108,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncCommittee: { AuthorizedRoles: flow.Roles(), - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, channels.SyncClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -112,11 +124,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncCommittee: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, channels.SyncClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -128,11 +140,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncCommittee: { AuthorizedRoles: flow.Roles(), - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, channels.SyncClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -144,11 +156,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncCommittee: { AuthorizedRoles: flow.Roles(), - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, channels.SyncClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -160,7 +172,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncCommittee: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -174,7 +186,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ConsensusClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -186,7 +198,19 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ConsensusClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, + }, + }, + } + authorizationConfigs[ClusterTimeoutObject] = MsgAuthConfig{ + Name: ClusterTimeoutObject, + Type: func() interface{} { + return new(messages.ClusterTimeoutObject) + }, + Config: map[channels.Channel]ChannelAuthConfig{ + channels.ConsensusClusterPrefix: { + AuthorizedRoles: flow.RoleList{flow.RoleCollection}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, }, } @@ -198,7 +222,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.SyncClusterPrefix: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -212,7 +236,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.PushGuarantees: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ReceiveGuarantees = PushGuarantees }, } @@ -224,7 +248,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.PushTransactions: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ReceiveTransactions = PushTransactions }, } @@ -238,7 +262,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.PushReceipts: { AuthorizedRoles: flow.RoleList{flow.RoleExecution}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ReceiveReceipts = PushReceipts }, } @@ -250,7 +274,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.PushApprovals: { AuthorizedRoles: flow.RoleList{flow.RoleVerification}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ReceiveApprovals = PushApprovals }, } @@ -264,7 +288,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.RequestChunks: { AuthorizedRoles: flow.RoleList{flow.RoleVerification}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias RequestChunks = ProvideChunks }, } @@ -276,7 +300,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ProvideChunks: { AuthorizedRoles: flow.RoleList{flow.RoleExecution}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, // channel alias RequestChunks = ProvideChunks }, } @@ -290,7 +314,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.RequestApprovalsByChunk: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolPublish}, + AllowedProtocols: Protocols{ProtocolTypePubSub}, }, // channel alias ProvideApprovalsByChunk = RequestApprovalsByChunk }, } @@ -302,7 +326,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ProvideApprovalsByChunk: { AuthorizedRoles: flow.RoleList{flow.RoleVerification}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, // channel alias ProvideApprovalsByChunk = RequestApprovalsByChunk }, } @@ -316,11 +340,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.RequestReceiptsByBlockID: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, channels.RequestCollections: { AuthorizedRoles: flow.RoleList{flow.RoleAccess, flow.RoleExecution}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -332,11 +356,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.ProvideReceiptsByBlockID: { AuthorizedRoles: flow.RoleList{flow.RoleExecution}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, channels.ProvideCollections: { AuthorizedRoles: flow.RoleList{flow.RoleCollection}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -350,11 +374,11 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.TestNetworkChannel: { AuthorizedRoles: flow.Roles(), - AllowedProtocols: Protocols{ProtocolPublish, ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypePubSub, ProtocolTypeUnicast}, }, channels.TestMetricsChannel: { AuthorizedRoles: flow.Roles(), - AllowedProtocols: Protocols{ProtocolPublish, ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypePubSub, ProtocolTypeUnicast}, }, }, } @@ -368,7 +392,7 @@ func initializeMessageAuthConfigsMap() { Config: map[channels.Channel]ChannelAuthConfig{ channels.DKGCommittee: { AuthorizedRoles: flow.RoleList{flow.RoleConsensus}, - AllowedProtocols: Protocols{ProtocolUnicast}, + AllowedProtocols: Protocols{ProtocolTypeUnicast}, }, }, } @@ -385,6 +409,8 @@ func GetMessageAuthConfig(v interface{}) (MsgAuthConfig, error) { return authorizationConfigs[BlockProposal], nil case *messages.BlockVote: return authorizationConfigs[BlockVote], nil + case *messages.TimeoutObject: + return authorizationConfigs[TimeoutObject], nil // protocol state sync case *messages.SyncRequest: @@ -403,6 +429,8 @@ func GetMessageAuthConfig(v interface{}) (MsgAuthConfig, error) { return authorizationConfigs[ClusterBlockProposal], nil case *messages.ClusterBlockVote: return authorizationConfigs[ClusterBlockVote], nil + case *messages.ClusterTimeoutObject: + return authorizationConfigs[ClusterTimeoutObject], nil case *messages.ClusterBlockResponse: return authorizationConfigs[ClusterBlockResponse], nil diff --git a/network/message/errors.go b/network/message/errors.go index 7d82b4f1e8b..5441027a7b0 100644 --- a/network/message/errors.go +++ b/network/message/errors.go @@ -31,3 +31,12 @@ func IsUnknownMsgTypeErr(err error) bool { var e UnknownMsgTypeErr return errors.As(err, &e) } + +// NewUnauthorizedProtocolError returns ErrUnauthorizedUnicastOnChannel or ErrUnauthorizedPublishOnChannel depending on the protocol provided. +func NewUnauthorizedProtocolError(p ProtocolType) error { + if p == ProtocolTypeUnicast { + return ErrUnauthorizedUnicastOnChannel + } + + return ErrUnauthorizedPublishOnChannel +} diff --git a/network/message/init.go b/network/message/init.go index 555e60d73e6..4bb3cb0dc60 100644 --- a/network/message/init.go +++ b/network/message/init.go @@ -46,6 +46,7 @@ func excludeConfig(name string, excludeList []string) bool { const ( BlockProposal = "BlockProposal" BlockVote = "BlockVote" + TimeoutObject = "Timeout" SyncRequest = "SyncRequest" SyncResponse = "SyncResponse" RangeRequest = "RangeRequest" @@ -53,6 +54,7 @@ const ( BlockResponse = "BlockResponse" ClusterBlockProposal = "ClusterBlockProposal" ClusterBlockVote = "ClusterBlockVote" + ClusterTimeoutObject = "ClusterTimeout" ClusterBlockResponse = "ClusterBlockResponse" CollectionGuarantee = "CollectionGuarantee" TransactionBody = "TransactionBody" diff --git a/network/message/protocols.go b/network/message/protocols.go index e70e1f84385..257bee15361 100644 --- a/network/message/protocols.go +++ b/network/message/protocols.go @@ -1,29 +1,25 @@ package message -type Protocol string - const ( - ProtocolPublish = Protocol("publish") - ProtocolUnicast = Protocol("unicast") + // ProtocolTypeUnicast is protocol type for unicast messages. + ProtocolTypeUnicast ProtocolType = "unicast" + // ProtocolTypePubSub is the protocol type for pubsub messages. + ProtocolTypePubSub ProtocolType = "pubsub" ) -func (p Protocol) String() string { - return string(p) -} - -// NewUnauthorizedProtocolError returns ErrUnauthorizedUnicastOnChannel or ErrUnauthorizedPublishOnChannel depending on the protocol provided. -func NewUnauthorizedProtocolError(p Protocol) error { - if p == ProtocolUnicast { - return ErrUnauthorizedUnicastOnChannel - } +// ProtocolType defines the type of the protocol a message is sent over. Currently, we have two types of protocols: +// - unicast: a message is sent to a single node through a direct connection. +// - pubsub: a message is sent to one or more nodes through a pubsub channel. +type ProtocolType string - return ErrUnauthorizedPublishOnChannel +func (m ProtocolType) String() string { + return string(m) } -type Protocols []Protocol +type Protocols []ProtocolType // Contains returns true if the protocol is in the list of Protocols. -func (pr Protocols) Contains(protocol Protocol) bool { +func (pr Protocols) Contains(protocol ProtocolType) bool { for _, p := range pr { if p == protocol { return true diff --git a/network/message_scope.go b/network/message_scope.go index 470fb30527e..3db13a1b2bd 100644 --- a/network/message_scope.go +++ b/network/message_scope.go @@ -15,31 +15,14 @@ const ( eventIDPackingPrefix = "libp2ppacking" ) -// ProtocolType defines the type of the protocol a message is sent over. Currently, we have two types of protocols: -// - unicast: a message is sent to a single node through a direct connection. -// - pubsub: a message is sent to a set of nodes through a pubsub channel. -type ProtocolType string - -func (m ProtocolType) String() string { - return string(m) -} - -const ( - // ProtocolTypeUnicast is protocol type for unicast messages. - ProtocolTypeUnicast ProtocolType = "unicast" - - // ProtocolTypePubSub is the protocol type for pubsub messages. - ProtocolTypePubSub ProtocolType = "pubsub" -) - // IncomingMessageScope captures the context around an incoming message that is received by the network layer. type IncomingMessageScope struct { - originId flow.Identifier // the origin node ID. - targetIds flow.IdentifierList // the target node IDs (i.e., intended recipients). - eventId hash.Hash // hash of the payload and channel. - msg *message.Message // the raw message received. - decodedPayload interface{} // decoded payload of the message. - protocol ProtocolType // the type of protocol used to receive the message. + originId flow.Identifier // the origin node ID. + targetIds flow.IdentifierList // the target node IDs (i.e., intended recipients). + eventId hash.Hash // hash of the payload and channel. + msg *message.Message // the raw message received. + decodedPayload interface{} // decoded payload of the message. + protocol message.ProtocolType // the type of protocol used to receive the message. } // NewIncomingScope creates a new incoming message scope. @@ -47,7 +30,7 @@ type IncomingMessageScope struct { // safe to crash the node when receiving a message. // It errors if event id (i.e., hash of the payload and channel) cannot be computed, or if it fails to // convert the target IDs from bytes slice to a flow.IdentifierList. -func NewIncomingScope(originId flow.Identifier, protocol ProtocolType, msg *message.Message, decodedPayload interface{}) (*IncomingMessageScope, error) { +func NewIncomingScope(originId flow.Identifier, protocol message.ProtocolType, msg *message.Message, decodedPayload interface{}) (*IncomingMessageScope, error) { eventId, err := EventId(channels.Channel(msg.ChannelID), msg.Payload) if err != nil { return nil, fmt.Errorf("could not compute event id: %w", err) @@ -79,7 +62,7 @@ func (m IncomingMessageScope) DecodedPayload() interface{} { return m.decodedPayload } -func (m IncomingMessageScope) Protocol() ProtocolType { +func (m IncomingMessageScope) Protocol() message.ProtocolType { return m.protocol } @@ -110,7 +93,7 @@ type OutgoingMessageScope struct { payload interface{} // the payload to be sent. encoder func(interface{}) ([]byte, error) // the encoder to encode the payload. msg *message.Message // raw proto message sent on wire. - protocol ProtocolType // the type of protocol used to send the message. + protocol message.ProtocolType // the type of protocol used to send the message. } // NewOutgoingScope creates a new outgoing message scope. @@ -123,7 +106,7 @@ func NewOutgoingScope( channelId channels.Channel, payload interface{}, encoder func(interface{}) ([]byte, error), - protocolType ProtocolType) (*OutgoingMessageScope, error) { + protocolType message.ProtocolType) (*OutgoingMessageScope, error) { scope := &OutgoingMessageScope{ targetIds: targetIds, channelId: channelId, @@ -132,13 +115,13 @@ func NewOutgoingScope( protocol: protocolType, } - if protocolType == ProtocolTypeUnicast { + if protocolType == message.ProtocolTypeUnicast { // for unicast messages, we should have exactly one target. if len(targetIds) != 1 { return nil, fmt.Errorf("expected exactly one target id for unicast message, got: %d", len(targetIds)) } } - if protocolType == ProtocolTypePubSub { + if protocolType == message.ProtocolTypePubSub { // for pubsub messages, we should have at least one target. if len(targetIds) == 0 { return nil, fmt.Errorf("expected at least one target id for pubsub message, got: %d", len(targetIds)) diff --git a/network/p2p/cache/node_blocklist_distributor.go b/network/p2p/cache/node_blocklist_distributor.go new file mode 100644 index 00000000000..7390fdc2850 --- /dev/null +++ b/network/p2p/cache/node_blocklist_distributor.go @@ -0,0 +1,36 @@ +package cache + +import ( + "sync" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network/p2p" +) + +// NodeBlockListDistributor subscribes to changes in the NodeBlocklistWrapper block list. +type NodeBlockListDistributor struct { + nodeBlockListConsumers []p2p.NodeBlockListConsumer + lock sync.RWMutex +} + +var _ p2p.NodeBlockListConsumer = (*NodeBlockListDistributor)(nil) + +func NewNodeBlockListDistributor() *NodeBlockListDistributor { + return &NodeBlockListDistributor{ + nodeBlockListConsumers: make([]p2p.NodeBlockListConsumer, 0), + } +} + +func (n *NodeBlockListDistributor) AddConsumer(consumer p2p.NodeBlockListConsumer) { + n.lock.Lock() + defer n.lock.Unlock() + n.nodeBlockListConsumers = append(n.nodeBlockListConsumers, consumer) +} + +func (n *NodeBlockListDistributor) OnNodeBlockListUpdate(blockList flow.IdentifierList) { + n.lock.RLock() + defer n.lock.RUnlock() + for _, consumer := range n.nodeBlockListConsumers { + consumer.OnNodeBlockListUpdate(blockList) + } +} diff --git a/network/p2p/cache/node_blocklist_wrapper.go b/network/p2p/cache/node_blocklist_wrapper.go index 5a8bff64da7..3dba3839b9b 100644 --- a/network/p2p/cache/node_blocklist_wrapper.go +++ b/network/p2p/cache/node_blocklist_wrapper.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/storage/badger/operation" ) @@ -38,13 +39,14 @@ type NodeBlocklistWrapper struct { identityProvider module.IdentityProvider blocklist IdentifierSet // `IdentifierSet` is a map, hence efficient O(1) lookup + notifier p2p.NodeBlockListConsumer } var _ module.IdentityProvider = (*NodeBlocklistWrapper)(nil) // NewNodeBlocklistWrapper wraps the given `IdentityProvider`. The blocklist is // loaded from the database (or assumed to be empty if no database entry is present). -func NewNodeBlocklistWrapper(identityProvider module.IdentityProvider, db *badger.DB) (*NodeBlocklistWrapper, error) { +func NewNodeBlocklistWrapper(identityProvider module.IdentityProvider, db *badger.DB, notifier p2p.NodeBlockListConsumer) (*NodeBlocklistWrapper, error) { blocklist, err := retrieveBlocklist(db) if err != nil { return nil, fmt.Errorf("failed to read set of blocked node IDs from data base: %w", err) @@ -54,6 +56,7 @@ func NewNodeBlocklistWrapper(identityProvider module.IdentityProvider, db *badge db: db, identityProvider: identityProvider, blocklist: blocklist, + notifier: notifier, }, nil } @@ -73,6 +76,8 @@ func (w *NodeBlocklistWrapper) Update(blocklist flow.IdentifierList) error { return fmt.Errorf("failed to persist set of blocked nodes to the data base: %w", err) } w.blocklist = b + w.notifier.OnNodeBlockListUpdate(blocklist) + return nil } @@ -111,7 +116,7 @@ func (w *NodeBlocklistWrapper) Identities(filter flow.IdentityFilter) flow.Ident w.m.RLock() for _, identity := range identities { if w.blocklist.Contains(identity.NodeID) { - var i flow.Identity = *identity // shallow copy is sufficient, because `Ejected` flag is in top-level struct + var i = *identity // shallow copy is sufficient, because `Ejected` flag is in top-level struct i.Ejected = true if filter(&i) { // we need to check the filter here again, because the filter might drop ejected nodes and we are modifying the ejected status here idtx = append(idtx, &i) @@ -154,7 +159,7 @@ func (w *NodeBlocklistWrapper) setEjectedIfBlocked(identity *flow.Identity) *flo // For blocked nodes, we want to return their `Identity` with the `Ejected` flag // set to true. Caution: we need to copy the `Identity` before we override `Ejected`, as we // would otherwise potentially change the wrapped IdentityProvider. - var i flow.Identity = *identity // shallow copy is sufficient, because `Ejected` flag is in top-level struct + var i = *identity // shallow copy is sufficient, because `Ejected` flag is in top-level struct i.Ejected = true return &i } diff --git a/network/p2p/cache/node_blocklist_wrapper_test.go b/network/p2p/cache/node_blocklist_wrapper_test.go index d1fd4330971..b42b9eac15f 100644 --- a/network/p2p/cache/node_blocklist_wrapper_test.go +++ b/network/p2p/cache/node_blocklist_wrapper_test.go @@ -16,6 +16,7 @@ import ( mocks "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/cache" + mockp2p "github.com/onflow/flow-go/network/p2p/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -24,7 +25,8 @@ type NodeBlocklistWrapperTestSuite struct { DB *badger.DB provider *mocks.IdentityProvider - wrapper *cache.NodeBlocklistWrapper + wrapper *cache.NodeBlocklistWrapper + mockNotifier *mockp2p.NodeBlockListConsumer } func (s *NodeBlocklistWrapperTestSuite) SetupTest() { @@ -32,7 +34,8 @@ func (s *NodeBlocklistWrapperTestSuite) SetupTest() { s.provider = new(mocks.IdentityProvider) var err error - s.wrapper, err = cache.NewNodeBlocklistWrapper(s.provider, s.DB) + s.mockNotifier = mockp2p.NewNodeBlockListConsumer(s.T()) + s.wrapper, err = cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) } @@ -75,7 +78,7 @@ func (s *NodeBlocklistWrapperTestSuite) TestHonestNode() { }) } -// TestBlacklistedNode tests proper handling of identities _on_ the blocklist: +// TestDenylistedNode tests proper handling of identities _on_ the blocklist: // - For any identity `i` with `i.NodeID ∈ blocklist`, the returned identity // should have `i.Ejected` set to `true` (irrespective of the `Ejected` // flag's initial returned by the wrapped `IdentityProvider`). @@ -88,8 +91,9 @@ func (s *NodeBlocklistWrapperTestSuite) TestHonestNode() { // While returning (non-nil identity, false) is not a defined return value, // we expect the wrapper to nevertheless handle this case to increase its // generality. -func (s *NodeBlocklistWrapperTestSuite) TestBlacklistedNode() { +func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { blocklist := unittest.IdentityListFixture(11) + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist.NodeIDs()).Once() err := s.wrapper.Update(blocklist.NodeIDs()) require.NoError(s.T(), err) @@ -233,6 +237,7 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { // step 2: _after_ putting node on blocklist, // an Identity with `Ejected` equal to `true` should be returned + s.mockNotifier.On("OnNodeBlockListUpdate", flow.IdentifierList{originalIdentity.NodeID}).Once() err := s.wrapper.Update(flow.IdentifierList{originalIdentity.NodeID}) require.NoError(s.T(), err) @@ -246,6 +251,7 @@ func (s *NodeBlocklistWrapperTestSuite) TestBlocklistAddRemove() { // step 3: after removing the node from the blocklist, // an Identity with `Ejected` equal to the original value should be returned + s.mockNotifier.On("OnNodeBlockListUpdate", flow.IdentifierList{}).Once() err = s.wrapper.Update(flow.IdentifierList{}) require.NoError(s.T(), err) @@ -272,18 +278,22 @@ func (s *NodeBlocklistWrapperTestSuite) TestUpdate() { blocklist2 := unittest.IdentifierListFixture(11) blocklist3 := unittest.IdentifierListFixture(5) + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist1).Once() err := s.wrapper.Update(blocklist1) require.NoError(s.T(), err) require.Equal(s.T(), blocklist1.Lookup(), s.wrapper.GetBlocklist().Lookup()) + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist2).Once() err = s.wrapper.Update(blocklist2) require.NoError(s.T(), err) require.Equal(s.T(), blocklist2.Lookup(), s.wrapper.GetBlocklist().Lookup()) + s.mockNotifier.On("OnNodeBlockListUpdate", (flow.IdentifierList)(nil)).Once() err = s.wrapper.ClearBlocklist() require.NoError(s.T(), err) require.Empty(s.T(), s.wrapper.GetBlocklist()) + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist3).Once() err = s.wrapper.Update(blocklist3) require.NoError(s.T(), err) require.Equal(s.T(), blocklist3.Lookup(), s.wrapper.GetBlocklist().Lookup()) @@ -309,35 +319,39 @@ func (s *NodeBlocklistWrapperTestSuite) TestDataBasePersist() { }) s.Run("Clear blocklist on empty database", func() { + s.mockNotifier.On("OnNodeBlockListUpdate", (flow.IdentifierList)(nil)).Once() err := s.wrapper.ClearBlocklist() // No-op as data base does not contain any block list require.NoError(s.T(), err) require.Empty(s.T(), s.wrapper.GetBlocklist()) // newly created wrapper should read `blocklist` from data base during initialization - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Empty(s.T(), w.GetBlocklist()) }) s.Run("Update blocklist and init new wrapper from database", func() { + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist).Once() err := s.wrapper.Update(blocklist) require.NoError(s.T(), err) // newly created wrapper should read `blocklist` from data base during initialization - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Equal(s.T(), blocklist.Lookup(), w.GetBlocklist().Lookup()) }) s.Run("Update and overwrite blocklist and then init new wrapper from database", func() { + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist).Once() err := s.wrapper.Update(blocklist) require.NoError(s.T(), err) + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist2).Once() err = s.wrapper.Update(blocklist2) require.NoError(s.T(), err) // newly created wrapper should read initial state from data base - w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Equal(s.T(), blocklist2.Lookup(), w.GetBlocklist().Lookup()) }) @@ -345,28 +359,31 @@ func (s *NodeBlocklistWrapperTestSuite) TestDataBasePersist() { s.Run("Update & clear & update and then init new wrapper from database", func() { // set blocklist -> // newly created wrapper should now read this list from data base during initialization + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist).Once() err := s.wrapper.Update(blocklist) require.NoError(s.T(), err) - w0, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w0, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Equal(s.T(), blocklist.Lookup(), w0.GetBlocklist().Lookup()) // clear blocklist -> // newly created wrapper should now read empty blocklist from data base during initialization + s.mockNotifier.On("OnNodeBlockListUpdate", (flow.IdentifierList)(nil)).Once() err = s.wrapper.ClearBlocklist() require.NoError(s.T(), err) - w1, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w1, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Empty(s.T(), w1.GetBlocklist()) // set blocklist2 -> // newly created wrapper should now read this list from data base during initialization + s.mockNotifier.On("OnNodeBlockListUpdate", blocklist2).Once() err = s.wrapper.Update(blocklist2) require.NoError(s.T(), err) - w2, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB) + w2, err := cache.NewNodeBlocklistWrapper(s.provider, s.DB, s.mockNotifier) require.NoError(s.T(), err) require.Equal(s.T(), blocklist2.Lookup(), w2.GetBlocklist().Lookup()) }) diff --git a/network/p2p/conduit/conduit.go b/network/p2p/conduit/conduit.go index 7034dbe6d58..353e67c29fc 100644 --- a/network/p2p/conduit/conduit.go +++ b/network/p2p/conduit/conduit.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" ) @@ -13,11 +15,23 @@ import ( // It directly passes the incoming messages to the corresponding methods of the // network Adapter. type DefaultConduitFactory struct { + *component.ComponentManager adapter network.Adapter } func NewDefaultConduitFactory() *DefaultConduitFactory { - return &DefaultConduitFactory{} + d := &DefaultConduitFactory{} + // worker added so conduit factory doesn't immediately shut down when it's started + cm := component.NewComponentManagerBuilder(). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + <-ctx.Done() + }).Build() + + d.ComponentManager = cm + + return d } // RegisterAdapter sets the Adapter component of the factory. diff --git a/network/p2p/connection/connManager.go b/network/p2p/connection/connManager.go index 2b140679616..9483da30d75 100644 --- a/network/p2p/connection/connManager.go +++ b/network/p2p/connection/connManager.go @@ -1,142 +1,148 @@ package connection import ( - "sync" + "context" + "fmt" + "time" "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" + libp2pconnmgr "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/rs/zerolog" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/p2p/connection/internal" ) -// ConnManager provides an implementation of Libp2p's ConnManager interface (https://godoc.org/github.com/libp2p/go-libp2p-core/connmgr#ConnManager) +const ( + // defaultHighWatermark is the default value for the high watermark (i.e., max number of connections). + // We assume a complete topology graph with maximum of 500 nodes. + defaultHighWatermark = 500 + + // defaultLowWatermark is the default value for the low watermark (i.e., min number of connections). + // We assume a complete topology graph with minimum of 450 nodes. + defaultLowWatermark = 450 + + // defaultGracePeriod is the default value for the grace period (i.e., time to wait before pruning a new connection). + defaultGracePeriod = 1 * time.Minute + + // defaultSilencePeriod is the default value for the silence period (i.e., time to wait before start pruning connections). + defaultSilencePeriod = 10 * time.Second +) + +// DefaultConnManagerConfig returns the default configuration for the connection manager. +func DefaultConnManagerConfig() *ManagerConfig { + return &ManagerConfig{ + HighWatermark: defaultHighWatermark, + LowWatermark: defaultLowWatermark, + GracePeriod: defaultGracePeriod, + SilencePeriod: defaultSilencePeriod, + } +} + +// ConnManager provides an implementation of Libp2p's ConnManager interface (https://pkg.go.dev/github.com/libp2p/go-libp2p/core/connmgr#ConnManager) // It is called back by libp2p when certain events occur such as opening/closing a stream, opening/closing connection etc. -// This implementation updates networking metrics when a peer connection is added or removed +// Current implementation primarily acts as a wrapper around libp2p's BasicConnMgr (https://pkg.go.dev/github.com/libp2p/go-libp2p/p2p/net/connmgr#BasicConnMgr). +// However, we override the notifiee callback to add additional functionality so that it provides metrics and logging instrumentation for Flow. type ConnManager struct { - connmgr.NullConnMgr // a null conn mgr provided by libp2p to allow implementing only the functions needed - n network.Notifiee // the notifiee callback provided by libp2p - log zerolog.Logger // logger to log connection, stream and other statistics about libp2p - metrics module.LibP2PConnectionMetrics + basicConnMgr *libp2pconnmgr.BasicConnMgr + n network.Notifiee // the notifiee callback provided by libp2p + log zerolog.Logger // logger to log connection, stream and other statistics about libp2p +} - plk sync.RWMutex - protected map[peer.ID]map[string]struct{} +var _ connmgr.ConnManager = (*ConnManager)(nil) + +type ManagerConfig struct { + // HighWatermark and LowWatermark govern the number of connections are maintained by the ConnManager. + // When the peer count exceeds the HighWatermark, as many peers will be pruned (and + // their connections terminated) until LowWatermark peers remain. In other words, whenever the + // peer count is x > HighWatermark, the ConnManager will prune x - LowWatermark peers. + // The pruning algorithm is as follows: + // 1. The ConnManager will not prune any peers that have been connected for less than GracePeriod. + // 2. The ConnManager will not prune any peers that are protected. + // 3. The ConnManager will sort the peers based on their number of streams and direction of connections, and + // prunes the peers with the least number of streams. If there are ties, the peer with the incoming connection + // will be pruned. If both peers have incoming connections, and there are still ties, one of the peers will be + // pruned at random. + // Algorithm implementation is in https://github.com/libp2p/go-libp2p/blob/master/p2p/net/connmgr/connmgr.go#L262-L318 + HighWatermark int // naming from libp2p + LowWatermark int // naming from libp2p + + // SilencePeriod is the time to wait before start pruning connections. + SilencePeriod time.Duration // naming from libp2p + // GracePeriod is the time to wait before pruning a new connection. + GracePeriod time.Duration // naming from libp2p } -func NewConnManager(log zerolog.Logger, metrics module.LibP2PConnectionMetrics) *ConnManager { +// NewConnManager creates a new connection manager. +// It errors if creating the basic connection manager of libp2p fails. +// The error is not benign, and we should crash the node if it happens. +// It is a malpractice to start the node without connection manager. +func NewConnManager(logger zerolog.Logger, metric module.LibP2PConnectionMetrics, cfg *ManagerConfig) (*ConnManager, error) { + basic, err := libp2pconnmgr.NewConnManager( + cfg.LowWatermark, + cfg.HighWatermark, + libp2pconnmgr.WithGracePeriod(cfg.GracePeriod), + libp2pconnmgr.WithSilencePeriod(cfg.SilencePeriod)) + if err != nil { + return nil, fmt.Errorf("failed to create basic connection manager of libp2p: %w", err) + } + cn := &ConnManager{ - log: log.With().Str("component", "conn_manager").Logger(), - NullConnMgr: connmgr.NullConnMgr{}, - metrics: metrics, - protected: make(map[peer.ID]map[string]struct{}), + log: logger.With().Str("component", "connection_manager").Logger(), + basicConnMgr: basic, } - n := &network.NotifyBundle{ListenCloseF: cn.ListenCloseNotifee, - ListenF: cn.ListenNotifee, - ConnectedF: cn.Connected, - DisconnectedF: cn.Disconnected} - cn.n = n - return cn + // aggregates the notifiee callbacks from libp2p and our own notifiee into one. + cn.n = internal.NewRelayNotifee(internal.NewLoggerNotifiee(cn.log, metric), cn.basicConnMgr.Notifee()) + + cn.log.Info(). + Int("low_watermark", cfg.LowWatermark). + Int("high_watermark", cfg.HighWatermark). + Dur("grace_period", cfg.GracePeriod). + Dur("silence_period", cfg.SilencePeriod). + Msg("connection manager initialized") + + return cn, nil } func (cm *ConnManager) Notifee() network.Notifiee { return cm.n } -// ListenNotifee is called by libp2p when network starts listening on an addr -func (cm *ConnManager) ListenNotifee(n network.Network, m multiaddr.Multiaddr) { - cm.log.Debug().Str("multiaddress", m.String()).Msg("listen started") +func (cm *ConnManager) Protect(id peer.ID, tag string) { + cm.basicConnMgr.Protect(id, tag) } -// called by libp2p when network stops listening on an addr -// * This is never called back by libp2p currently and may be a bug on their side -func (cm *ConnManager) ListenCloseNotifee(n network.Network, m multiaddr.Multiaddr) { - // just log the multiaddress on which we listen - cm.log.Debug().Str("multiaddress", m.String()).Msg("listen stopped ") +func (cm *ConnManager) Unprotect(id peer.ID, tag string) bool { + return cm.basicConnMgr.Unprotect(id, tag) } -// Connected is called by libp2p when a connection opened -func (cm *ConnManager) Connected(n network.Network, con network.Conn) { - cm.logConnectionUpdate(n, con, "connection established") - cm.updateConnectionMetric(n) +func (cm *ConnManager) IsProtected(id peer.ID, tag string) bool { + return cm.basicConnMgr.IsProtected(id, tag) } -// Disconnected is called by libp2p when a connection closed -func (cm *ConnManager) Disconnected(n network.Network, con network.Conn) { - cm.logConnectionUpdate(n, con, "connection removed") - cm.updateConnectionMetric(n) +func (cm *ConnManager) TagPeer(id peer.ID, s string, i int) { + cm.basicConnMgr.TagPeer(id, s, i) } -func (cm *ConnManager) updateConnectionMetric(n network.Network) { - var totalInbound uint = 0 - var totalOutbound uint = 0 - - for _, conn := range n.Conns() { - switch conn.Stat().Direction { - case network.DirInbound: - totalInbound++ - case network.DirOutbound: - totalOutbound++ - } - } - - cm.metrics.InboundConnections(totalInbound) - cm.metrics.OutboundConnections(totalOutbound) +func (cm *ConnManager) UntagPeer(p peer.ID, tag string) { + cm.basicConnMgr.UntagPeer(p, tag) } -func (cm *ConnManager) logConnectionUpdate(n network.Network, con network.Conn, logMsg string) { - cm.log.Debug(). - Str("remote_peer", con.RemotePeer().String()). - Str("remote_addrs", con.RemoteMultiaddr().String()). - Str("local_peer", con.LocalPeer().String()). - Str("local_addrs", con.LocalMultiaddr().String()). - Str("direction", con.Stat().Direction.String()). - Int("total_connections", len(n.Conns())). - Msg(logMsg) +func (cm *ConnManager) UpsertTag(p peer.ID, tag string, upsert func(int) int) { + cm.basicConnMgr.UpsertTag(p, tag, upsert) } -func (cm *ConnManager) Protect(id peer.ID, tag string) { - cm.plk.Lock() - defer cm.plk.Unlock() - - tags, ok := cm.protected[id] - if !ok { - tags = make(map[string]struct{}, 2) - cm.protected[id] = tags - } - tags[tag] = struct{}{} +func (cm *ConnManager) GetTagInfo(p peer.ID) *connmgr.TagInfo { + return cm.basicConnMgr.GetTagInfo(p) } -func (cm *ConnManager) Unprotect(id peer.ID, tag string) (protected bool) { - cm.plk.Lock() - defer cm.plk.Unlock() - - tags, ok := cm.protected[id] - if !ok { - return false - } - if delete(tags, tag); len(tags) == 0 { - delete(cm.protected, id) - return false - } - return true +func (cm *ConnManager) TrimOpenConns(ctx context.Context) { + cm.basicConnMgr.TrimOpenConns(ctx) } -func (cm *ConnManager) IsProtected(id peer.ID, tag string) (protected bool) { - cm.plk.RLock() - defer cm.plk.RUnlock() - - tags, ok := cm.protected[id] - if !ok { - return false - } - - if tag == "" { - return true - } - - _, protected = tags[tag] - return protected +func (cm *ConnManager) Close() error { + return cm.basicConnMgr.Close() } diff --git a/network/p2p/connection/connManager_test.go b/network/p2p/connection/connManager_test.go index ac86e977d0d..33808381de0 100644 --- a/network/p2p/connection/connManager_test.go +++ b/network/p2p/connection/connManager_test.go @@ -1,16 +1,20 @@ package connection_test import ( + "context" "fmt" "os" "testing" + "time" "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/p2p/connection" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/module/metrics" @@ -45,13 +49,14 @@ var isNotProtected = fun{ false, } -// TestConnectionManagerProtection tests that multiple protect and unprotect calls result in the correct IsProtected +// TestConnectionManagerProtection tests that multiple protected and unprotected calls result in the correct IsProtected // status for a peer ID func TestConnectionManagerProtection(t *testing.T) { log := zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) noopMetrics := metrics.NewNoopCollector() - connManager := connection.NewConnManager(log, noopMetrics) + connManager, err := connection.NewConnManager(log, noopMetrics, connection.DefaultConnManagerConfig()) + require.NoError(t, err) testCases := [][]fun{ // single stream created on a connection @@ -88,3 +93,71 @@ func generatePeerInfo(t *testing.T) peer.ID { require.NoError(t, err) return pInfo.ID } + +// TestConnectionManager_Watermarking tests that the connection manager prunes connections when the number of connections +// exceeds the high watermark and that it does not prune connections when the number of connections is below the low watermark. +func TestConnectionManager_Watermarking(t *testing.T) { + sporkId := unittest.IdentifierFixture() + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + defer cancel() + + cfg := &connection.ManagerConfig{ + HighWatermark: 4, // whenever the number of connections exceeds 4, connection manager prune connections. + LowWatermark: 2, // connection manager prune connections until the number of connections is 2. + GracePeriod: 500 * time.Millisecond, // extra connections will be pruned if they are older than a second (just for testing). + SilencePeriod: time.Second, // connection manager prune checking kicks in every 5 seconds (just for testing). + } + thisConnMgr, err := connection.NewConnManager( + unittest.Logger(), + metrics.NewNoopCollector(), + cfg) + require.NoError(t, err) + + thisNode, _ := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + p2ptest.WithConnectionManager(thisConnMgr)) + + otherNodes, _ := p2ptest.NodesFixture(t, sporkId, t.Name(), 5) + + nodes := append(otherNodes, thisNode) + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) + + // connect this node to all other nodes. + for _, otherNode := range otherNodes { + require.NoError(t, thisNode.Host().Connect(ctx, otherNode.Host().Peerstore().PeerInfo(otherNode.Host().ID()))) + } + + // ensures this node is connected to all other nodes (based on the number of connections). + require.Eventuallyf(t, func() bool { + return len(thisNode.Host().Network().Conns()) == len(otherNodes) + }, 1*time.Second, 100*time.Millisecond, "expected %d connections, got %d", len(otherNodes), len(thisNode.Host().Network().Conns())) + + // wait for grace period to expire and connection manager kick in as the number of connections is beyond high watermark. + time.Sleep(time.Second) + + // ensures that eventually connection manager closes connections till the low watermark is reached. + require.Eventuallyf(t, func() bool { + return len(thisNode.Host().Network().Conns()) == cfg.LowWatermark + }, 1*time.Second, 100*time.Millisecond, "expected %d connections, got %d", cfg.LowWatermark, len(thisNode.Host().Network().Conns())) + + // connects this node to one of the other nodes that is pruned by connection manager. + for _, otherNode := range otherNodes { + if len(thisNode.Host().Network().ConnsToPeer(otherNode.Host().ID())) == 0 { + require.NoError(t, thisNode.Host().Connect(ctx, otherNode.Host().Peerstore().PeerInfo(otherNode.Host().ID()))) + break // we only need to connect to one node. + } + } + + // wait for another grace period to expire and connection manager kick in. + time.Sleep(time.Second) + + // ensures that connection manager does not close any connections as the number of connections is below low watermark. + require.Eventuallyf(t, func() bool { + return len(thisNode.Host().Network().Conns()) == cfg.LowWatermark+1 + }, 1*time.Second, 100*time.Millisecond, "expected %d connections, got %d", cfg.LowWatermark+1, len(thisNode.Host().Network().Conns())) +} diff --git a/network/p2p/connection/connection_gater.go b/network/p2p/connection/connection_gater.go index f26127b9e85..2ee0df16331 100644 --- a/network/p2p/connection/connection_gater.go +++ b/network/p2p/connection/connection_gater.go @@ -10,6 +10,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" + "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/logging" ) @@ -42,12 +43,18 @@ type ConnGater struct { sync.RWMutex onInterceptPeerDialFilters []p2p.PeerFilter onInterceptSecuredFilters []p2p.PeerFilter - log zerolog.Logger + + // identityProvider provides the identity of a node given its peer ID for logging purposes only. + // It is not used for allowlisting or filtering. We use the onInterceptPeerDialFilters and onInterceptSecuredFilters + // to determine if a node should be allowed to connect. + identityProvider module.IdentityProvider + log zerolog.Logger } -func NewConnGater(log zerolog.Logger, opts ...ConnGaterOption) *ConnGater { +func NewConnGater(log zerolog.Logger, identityProvider module.IdentityProvider, opts ...ConnGaterOption) *ConnGater { cg := &ConnGater{ - log: log.With().Str("component", "connection_gater").Logger(), + log: log.With().Str("component", "connection_gater").Logger(), + identityProvider: identityProvider, } for _, opt := range opts { @@ -59,22 +66,36 @@ func NewConnGater(log zerolog.Logger, opts ...ConnGaterOption) *ConnGater { // InterceptPeerDial - a callback which allows or disallows outbound connection func (c *ConnGater) InterceptPeerDial(p peer.ID) bool { + lg := c.log.With().Str("peer_id", p.String()).Logger() + if len(c.onInterceptPeerDialFilters) == 0 { - c.log.Debug(). - Str("peer_id", p.String()). - Msg("allowing outbound connection intercept peer dial has no peer filters set") + lg.Warn(). + Msg("outbound connection established with no intercept peer dial filters") return true } + identity, ok := c.identityProvider.ByPeerID(p) + if !ok { + lg = lg.With(). + Str("remote_node_id", "unknown"). + Str("role", "unknown"). + Logger() + } else { + lg = lg.With(). + Hex("remote_node_id", logging.ID(identity.NodeID)). + Str("role", identity.Role.String()). + Logger() + } + if err := c.peerIDPassesAllFilters(p, c.onInterceptPeerDialFilters); err != nil { // log the filtered outbound connection attempt - c.log.Warn(). + lg.Warn(). Err(err). - Str("peer_id", p.String()). Msg("rejected outbound connection attempt") return false } + lg.Info().Msg("outbound connection established") return true } @@ -99,10 +120,23 @@ func (c *ConnGater) InterceptSecured(dir network.Direction, p peer.ID, addr netw Logger() if len(c.onInterceptSecuredFilters) == 0 { - lg.Info().Msg("inbound connection established") + lg.Warn().Msg("inbound connection established with no intercept secured filters") return true } + identity, ok := c.identityProvider.ByPeerID(p) + if !ok { + lg = lg.With(). + Str("remote_node_id", "unknown"). + Str("role", "unknown"). + Logger() + } else { + lg = lg.With(). + Hex("remote_node_id", logging.ID(identity.NodeID)). + Str("role", identity.Role.String()). + Logger() + } + if err := c.peerIDPassesAllFilters(p, c.onInterceptSecuredFilters); err != nil { // log the illegal connection attempt from the remote node lg.Error(). diff --git a/network/p2p/connection/connection_gater_test.go b/network/p2p/connection/connection_gater_test.go index 7634e4347df..9bf191218c2 100644 --- a/network/p2p/connection/connection_gater_test.go +++ b/network/p2p/connection/connection_gater_test.go @@ -16,6 +16,8 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/testutils" @@ -31,31 +33,33 @@ func TestConnectionGating(t *testing.T) { signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) sporkID := unittest.IdentifierFixture() - + idProvider := mockmodule.NewIdentityProvider(t) // create 2 nodes node1Peers := unittest.NewProtectedMap[peer.ID, struct{}]() node1, node1Id := p2ptest.NodeFixture( t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(func(p peer.ID) error { + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { if !node1Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } return nil }))) + idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() node2Peers := unittest.NewProtectedMap[peer.ID, struct{}]() node2, node2Id := p2ptest.NodeFixture( t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(func(p peer.ID) error { + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { if !node2Peers.Has(p) { return fmt.Errorf("id not found: %s", p.String()) } return nil }))) + idProvider.On("ByPeerID", node2.Host().ID()).Return(&node2Id, true).Maybe() nodes := []p2p.LibP2PNode{node1, node2} ids := flow.IdentityList{&node1Id, &node2Id} @@ -80,12 +84,15 @@ func TestConnectionGating(t *testing.T) { // add node2 to node1's allow list, but not the other way around. node1Peers.Add(node2.Host().ID(), struct{}{}) + // from node2 -> node1 should also NOT work, since node 1 is not in node2's allow list for dialing! + p2pfixtures.EnsureNoStreamCreation(t, ctx, []p2p.LibP2PNode{node2}, []p2p.LibP2PNode{node1}, func(t *testing.T, err error) { + // dialing node-1 by node-2 should fail locally at the connection gater of node-2. + require.True(t, errors.Is(err, swarm.ErrGaterDisallowedConnection)) + }) + // now node2 should be able to connect to node1. // from node1 -> node2 shouldn't work p2pfixtures.EnsureNoStreamCreation(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}) - - // however, from node2 -> node1 should also NOT work, since node 1 is not in node2's allow list for dialing! - p2pfixtures.EnsureNoStreamCreation(t, ctx, []p2p.LibP2PNode{node2}, []p2p.LibP2PNode{node1}) }) t.Run("outbound connection to an approved node is allowed", func(t *testing.T) { @@ -98,6 +105,117 @@ func TestConnectionGating(t *testing.T) { }) } +// TestConnectionGating_ResourceAllocation_AllowListing tests resource allocation when a connection from an allow-listed node is established. +// The test directly mocks the underlying resource manager metrics of the libp2p native resource manager to ensure that the +// expected set of resources are allocated for the connection upon establishment. +func TestConnectionGating_ResourceAllocation_AllowListing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + sporkID := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) + + node1, node1Id := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + p2ptest.WithRole(flow.RoleConsensus)) + + node2Metrics := mockmodule.NewLibP2PMetrics(t) + // libp2p native resource manager metrics: + // we expect exactly 1 connection to be established from node1 to node2 (inbound for node 2). + node2Metrics.On("AllowConn", network.DirInbound, true).Return().Once() + // we expect the libp2p.identify service to be used to establish the connection. + node2Metrics.On("AllowService", "libp2p.identify").Return() + // we expect the node2 attaching node1 to the incoming connection. + node2Metrics.On("AllowPeer", node1.Host().ID()).Return() + // we expect node2 allocate memory for the incoming connection. + node2Metrics.On("AllowMemory", mock.Anything) + // we expect node2 to allow the stream to be created. + node2Metrics.On("AllowStream", node1.Host().ID(), mock.Anything) + // we expect node2 to attach protocol to the created stream. + node2Metrics.On("AllowProtocol", mock.Anything).Return() + + // Flow-level resource allocation metrics: + // We expect both of the following to be called as they are called together in the same function. + node2Metrics.On("InboundConnections", mock.Anything).Return() + node2Metrics.On("OutboundConnections", mock.Anything).Return() + + // we create node2 with a connection gater that allows all connections and the mocked metrics collector. + node2, node2Id := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + p2ptest.WithRole(flow.RoleConsensus), + p2ptest.WithMetricsCollector(node2Metrics), + // we use default resource manager rather than the test resource manager to ensure that the metrics are called. + p2ptest.WithDefaultResourceManager(), + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + return nil // allow all connections. + }))) + idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() + idProvider.On("ByPeerID", node2.Host().ID()).Return(&node2Id, true).Maybe() + + nodes := []p2p.LibP2PNode{node1, node2} + ids := flow.IdentityList{&node1Id, &node2Id} + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) + + p2pfixtures.AddNodesToEachOthersPeerStore(t, nodes, ids) + + // now node-1 should be able to connect to node-2. + p2pfixtures.EnsureStreamCreation(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}) + + node2Metrics.AssertExpectations(t) +} + +// TestConnectionGating_ResourceAllocation_DisAllowListing tests resource allocation when a connection from an allow-listed node is established. +func TestConnectionGating_ResourceAllocation_DisAllowListing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + sporkID := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) + + node1, node1Id := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + p2ptest.WithRole(flow.RoleConsensus)) + + node2Metrics := mockmodule.NewLibP2PMetrics(t) + node2Metrics.On("AllowConn", network.DirInbound, true).Return() + node2, node2Id := p2ptest.NodeFixture( + t, + sporkID, + t.Name(), + p2ptest.WithRole(flow.RoleConsensus), + p2ptest.WithMetricsCollector(node2Metrics), + // we use default resource manager rather than the test resource manager to ensure that the metrics are called. + p2ptest.WithDefaultResourceManager(), + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(p peer.ID) error { + return fmt.Errorf("disallowed connection") // rejecting all connections. + }))) + idProvider.On("ByPeerID", node1.Host().ID()).Return(&node1Id, true).Maybe() + idProvider.On("ByPeerID", node2.Host().ID()).Return(&node2Id, true).Maybe() + + nodes := []p2p.LibP2PNode{node1, node2} + ids := flow.IdentityList{&node1Id, &node2Id} + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) + + p2pfixtures.AddNodesToEachOthersPeerStore(t, nodes, ids) + + // now node2 should be able to connect to node1. + p2pfixtures.EnsureNoStreamCreation(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}) + + // as node-2 connection gater is rejecting all connections, none of the following resource allocation methods should be called. + node2Metrics.AssertNotCalled(t, "AllowService", mock.Anything) // no service is allowed, e.g., libp2p.identify + node2Metrics.AssertNotCalled(t, "AllowPeer", mock.Anything) // no peer is allowed to be attached to the connection. + node2Metrics.AssertNotCalled(t, "AllowMemory", mock.Anything) // no memory is EVER allowed to be used during the test. + node2Metrics.AssertNotCalled(t, "AllowStream", mock.Anything, mock.Anything) // no stream is allowed to be created. +} + // TestConnectionGater_InterceptUpgrade tests the connection gater only upgrades the connections to the allow-listed peers. // Upgrading a connection means that the connection is the last phase of the connection establishment process. // It means that the connection is ready to be used for sending and receiving messages. @@ -161,22 +279,18 @@ func TestConnectionGater_InterceptUpgrade(t *testing.T) { p2ptest.StartNodes(t, signalerCtx, nodes, 1*time.Second) defer p2ptest.StopNodes(t, nodes, cancel, 1*time.Second) - ensureCommunicationSilenceAmongGroups(t, ctx, sporkId, nodes[:1], nodes[1:]) - - // Checks that only the allowed nodes can establish an upgradable connection. - // We intentionally mock this after checking for communication silence. - // As no connection to/from a disallowed node should ever reach the upgradable connection stage. + // Checks that only an allowed REMOTE node can establish an upgradable connection. connectionGater.On("InterceptUpgraded", mock.Anything).Run(func(args mock.Arguments) { conn, ok := args.Get(0).(network.Conn) require.True(t, ok) + // we don't check for the local peer as with v0.24 of libp2p, the local peer may be able to upgrade an empty connection + // even though the remote peer has already disconnected and rejected the connection. remote := conn.RemotePeer() require.False(t, disallowedPeerIds.Has(remote)) - - local := conn.LocalPeer() - require.False(t, disallowedPeerIds.Has(local)) }).Return(true, control.DisconnectReason(0)) + ensureCommunicationSilenceAmongGroups(t, ctx, sporkId, nodes[:1], nodes[1:]) ensureCommunicationOverAllProtocols(t, ctx, sporkId, nodes[1:], inbounds[1:]) } @@ -187,6 +301,7 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) sporkId := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) defer cancel() count := 5 @@ -219,7 +334,7 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { } return list }), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(func(pid peer.ID) error { + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { return disallowedList.ForEach(func(id *flow.Identity, _ struct{}) error { bid, err := unittest.PeerIDFromFlowID(id) require.NoError(t, err) @@ -229,6 +344,7 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { return nil }) }))) + idProvider.On("ByPeerID", node.Host().ID()).Return(&id, true).Maybe() nodes = append(nodes, node) ids = append(ids, &id) diff --git a/network/p2p/connection/internal/loggerNotifiee.go b/network/p2p/connection/internal/loggerNotifiee.go new file mode 100644 index 00000000000..9dc6fab9f75 --- /dev/null +++ b/network/p2p/connection/internal/loggerNotifiee.go @@ -0,0 +1,71 @@ +package internal + +import ( + "github.com/libp2p/go-libp2p/core/network" + "github.com/multiformats/go-multiaddr" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/module" +) + +type LoggerNotifiee struct { + logger zerolog.Logger + metrics module.LibP2PConnectionMetrics +} + +var _ network.Notifiee = (*LoggerNotifiee)(nil) + +func NewLoggerNotifiee(logger zerolog.Logger, metrics module.LibP2PConnectionMetrics) *LoggerNotifiee { + return &LoggerNotifiee{ + logger: logger, + metrics: metrics, + } +} + +func (l *LoggerNotifiee) Listen(_ network.Network, multiaddr multiaddr.Multiaddr) { + // just log the multiaddress on which we listen + l.logger.Info().Str("multiaddress", multiaddr.String()).Msg("listen started") +} + +func (l *LoggerNotifiee) ListenClose(_ network.Network, multiaddr multiaddr.Multiaddr) { + l.logger.Info().Str("multiaddress", multiaddr.String()).Msg("listen stopped") +} + +func (l *LoggerNotifiee) Connected(n network.Network, conn network.Conn) { + l.updateConnectionMetric(n) + lg := l.connectionUpdateLogger(n, conn) + lg.Info().Msg("connection established") +} + +func (l *LoggerNotifiee) Disconnected(n network.Network, conn network.Conn) { + l.updateConnectionMetric(n) + lg := l.connectionUpdateLogger(n, conn) + lg.Warn().Msg("connection closed") +} + +func (l *LoggerNotifiee) connectionUpdateLogger(n network.Network, con network.Conn) zerolog.Logger { + return l.logger.With(). + Str("remote_peer", con.RemotePeer().String()). + Str("remote_address", con.RemoteMultiaddr().String()). + Str("local_peer", con.LocalPeer().String()). + Str("local_address", con.LocalMultiaddr().String()). + Str("direction", con.Stat().Direction.String()). + Int("total_connections", len(n.Conns())).Logger() +} + +func (l *LoggerNotifiee) updateConnectionMetric(n network.Network) { + var totalInbound uint = 0 + var totalOutbound uint = 0 + + for _, conn := range n.Conns() { + switch conn.Stat().Direction { + case network.DirInbound: + totalInbound++ + case network.DirOutbound: + totalOutbound++ + } + } + + l.metrics.InboundConnections(totalInbound) + l.metrics.OutboundConnections(totalOutbound) +} diff --git a/network/p2p/connection/internal/relayNotifee.go b/network/p2p/connection/internal/relayNotifee.go new file mode 100644 index 00000000000..d85cb748dbe --- /dev/null +++ b/network/p2p/connection/internal/relayNotifee.go @@ -0,0 +1,42 @@ +package internal + +import ( + "github.com/libp2p/go-libp2p/core/network" + "github.com/multiformats/go-multiaddr" +) + +// RelayNotifee is a notifiee that relays notifications to a list of notifiees. +// A network.Notifiee is a function that is called when a network event occurs, such as a new connection. +type RelayNotifee struct { + n []network.Notifiee +} + +var _ network.Notifiee = (*RelayNotifee)(nil) + +func NewRelayNotifee(notifiees ...network.Notifiee) *RelayNotifee { + return &RelayNotifee{notifiees} +} + +func (r *RelayNotifee) Listen(n network.Network, multiaddr multiaddr.Multiaddr) { + for _, notifiee := range r.n { + notifiee.Listen(n, multiaddr) + } +} + +func (r *RelayNotifee) ListenClose(n network.Network, multiaddr multiaddr.Multiaddr) { + for _, notifiee := range r.n { + notifiee.ListenClose(n, multiaddr) + } +} + +func (r *RelayNotifee) Connected(n network.Network, conn network.Conn) { + for _, notifiee := range r.n { + notifiee.Connected(n, conn) + } +} + +func (r *RelayNotifee) Disconnected(n network.Network, conn network.Conn) { + for _, notifiee := range r.n { + notifiee.Disconnected(n, conn) + } +} diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index 4cb364cb456..ded4a58c746 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -7,11 +7,13 @@ import ( "sync" "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/utils/logging" ) // DefaultPeerUpdateInterval is default duration for which the peer manager waits in between attempts to update peer connections @@ -19,6 +21,7 @@ var DefaultPeerUpdateInterval = 10 * time.Minute var _ p2p.PeerManager = (*PeerManager)(nil) var _ component.Component = (*PeerManager)(nil) +var _ p2p.RateLimiterConsumer = (*PeerManager)(nil) // PeerManager adds and removes connections to peers periodically and on request type PeerManager struct { @@ -152,3 +155,15 @@ func (pm *PeerManager) SetPeersProvider(peersProvider p2p.PeersProvider) { pm.peersProvider = peersProvider } + +func (pm *PeerManager) OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) { + pm.logger.Warn(). + Str("peer_id", pid.String()). + Str("role", role). + Str("message_type", msgType). + Str("topic", topic). + Str("reason", reason). + Bool(logging.KeySuspicious, true). + Msg("pruning connection to rate-limited peer") + pm.RequestPeerUpdate() +} diff --git a/network/p2p/consumer.go b/network/p2p/consumer.go new file mode 100644 index 00000000000..580cb1a4a6b --- /dev/null +++ b/network/p2p/consumer.go @@ -0,0 +1,17 @@ +package p2p + +import ( + "github.com/onflow/flow-go/model/flow" +) + +// NodeBlockListConsumer consumes notifications from the cache.NodeBlocklistWrapper whenever the block list is updated. +// Implementations must: +// - be concurrency safe +// - be non-blocking +type NodeBlockListConsumer interface { + // OnNodeBlockListUpdate notifications whenever the node block list is updated. + // Prerequisites: + // Implementation must be concurrency safe; Non-blocking; + // and must handle repetition of the same events (with some processing overhead). + OnNodeBlockListUpdate(list flow.IdentifierList) +} diff --git a/network/p2p/dht/dht_test.go b/network/p2p/dht/dht_test.go index 5501b087c61..bc0cc970fd9 100644 --- a/network/p2p/dht/dht_test.go +++ b/network/p2p/dht/dht_test.go @@ -13,7 +13,6 @@ import ( libp2pmsg "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" @@ -158,7 +157,7 @@ func TestPubSubWithDHTDiscovery(t *testing.T) { logger := unittest.Logger() - topicValidator := flowpubsub.TopicValidator(logger, codec, unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()), unittest.AllowAllPeerFilter()) + topicValidator := flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter()) for _, n := range nodes { s, err := n.Subscribe(topic, topicValidator) require.NoError(t, err) diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index 414fb5b2f68..8b081c2f61f 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -65,7 +65,8 @@ const ( ) var ( - _ network.Middleware = (*Middleware)(nil) + _ network.Middleware = (*Middleware)(nil) + _ p2p.NodeBlockListConsumer = (*Middleware)(nil) // ErrUnicastMsgWithoutSub error is provided to the slashing violations consumer in the case where // the middleware receives a message via unicast but does not have a corresponding subscription for @@ -201,7 +202,6 @@ func NewMiddleware( }).Build() mw.Component = cm - return mw } @@ -319,7 +319,9 @@ func (m *Middleware) start(ctx context.Context) error { // topologyPeers callback used by the peer manager to get the list of peer ID's // which this node should be directly connected to as peers. The peer ID list -// returned will be filtered through any configured m.peerManagerFilters. +// returned will be filtered through any configured m.peerManagerFilters. If the +// underlying libp2p node has a peer manager configured this func will be used as the +// peers provider. func (m *Middleware) topologyPeers() peer.IDSlice { peerIDs := make([]peer.ID, 0) for _, id := range m.peerIDs(m.ov.Topology().NodeIDs()) { @@ -344,6 +346,16 @@ func (m *Middleware) topologyPeers() peer.IDSlice { return peerIDs } +// OnNodeBlockListUpdate removes all peers in the blocklist from the underlying libp2pnode. +func (m *Middleware) OnNodeBlockListUpdate(blockList flow.IdentifierList) { + for _, pid := range m.peerIDs(blockList) { + err := m.libP2PNode.RemovePeer(pid) + if err != nil { + m.log.Error().Err(err).Str("peer_id", pid.String()).Msg("failed to disconnect from blocklisted peer") + } + } +} + // SendDirect sends msg on a 1-1 direct connection to the target ID. It models a guaranteed delivery asynchronous // direct one-to-one connection on the underlying network. No intermediate node on the overlay is utilized // as the router. @@ -512,11 +524,18 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { // ignore messages if node does not have subscription to topic if !m.libP2PNode.HasSubscription(topic) { violation := &slashing.Violation{ - Identity: nil, PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolUnicast, + Identity: nil, PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolTypeUnicast, + } + + msgCode, err := codec.MessageCodeFromPayload(msg.Payload) + if err != nil { + violation.Err = err + m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + return } // msg type is not guaranteed to be correct since it is set by the client - _, what, err := codec.InterfaceFromMessageCode(msg.Payload[0]) + _, what, err := codec.InterfaceFromMessageCode(msgCode) if err != nil { violation.Err = err m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) @@ -583,7 +602,7 @@ func (m *Middleware) Subscribe(channel channels.Channel) error { peerFilter = m.isProtocolParticipant() } - topicValidator := flowpubsub.TopicValidator(m.log, m.codec, m.slashingViolationsConsumer, peerFilter, validators...) + topicValidator := flowpubsub.TopicValidator(m.log, peerFilter, validators...) s, err := m.libP2PNode.Subscribe(topic, topicValidator) if err != nil { return fmt.Errorf("could not subscribe to topic (%s): %w", topic, err) @@ -606,8 +625,8 @@ func (m *Middleware) Subscribe(channel channels.Channel) error { } // processPubSubMessages processes messages received from the pubsub subscription. -func (m *Middleware) processPubSubMessages(msg *message.Message, decodedMsgPayload interface{}, peerID peer.ID) { - m.processAuthenticatedMessage(msg, decodedMsgPayload, peerID, network.ProtocolTypePubSub) +func (m *Middleware) processPubSubMessages(msg *message.Message, peerID peer.ID) { + m.processAuthenticatedMessage(msg, peerID, message.ProtocolTypePubSub) } // Unsubscribe unsubscribes the middleware from a channel. @@ -632,42 +651,16 @@ func (m *Middleware) Unsubscribe(channel channels.Channel) error { // sent via unicast stream. This func should be invoked in a separate goroutine to avoid creating a message decoding bottleneck. func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *message.Message) { channel := channels.Channel(msg.ChannelID) - decodedMsgPayload, err := m.codec.Decode(msg.Payload) - if codec.IsErrUnknownMsgCode(err) { - // slash peer if message contains unknown message code byte - violation := &slashing.Violation{ - PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolUnicast, Err: err, - } - m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - return - } - if codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err) { - // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte - violation := &slashing.Violation{ - PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolUnicast, Err: err, - } - m.slashingViolationsConsumer.OnInvalidMsgError(violation) - return - } - // unexpected error condition. this indicates there's a bug - // don't crash as a result of external inputs since that creates a DoS vector. + // TODO: once we've implemented per topic message size limits per the TODO above, + // we can remove this check + maxSize, err := unicastMaxMsgSizeByCode(msg.Payload) if err != nil { - m.log. - Error(). - Err(fmt.Errorf("unexpected error while decoding message: %w", err)). - Str("peer_id", remotePeer.String()). - Str("channel", msg.ChannelID). - Bool(logging.KeySuspicious, true). - Msg("failed to decode message payload") + m.slashingViolationsConsumer.OnUnknownMsgTypeError(&slashing.Violation{ + Identity: nil, PeerID: remotePeer.String(), MsgType: "", Channel: channel, Protocol: message.ProtocolTypeUnicast, Err: err, + }) return } - - messageType := network.MessageType(decodedMsgPayload) - - // TODO: once we've implemented per topic message size limits per the TODO above, - // we can remove this check - maxSize := unicastMaxMsgSize(messageType) if msg.Size() > maxSize { // message size exceeded m.log.Error(). @@ -682,7 +675,7 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag // if message channel is not public perform authorized sender validation if !channels.IsPublicChannel(channel) { - _, err := m.authorizedSenderValidator.Validate(remotePeer, decodedMsgPayload, channel, message.ProtocolUnicast) + messageType, err := m.authorizedSenderValidator.Validate(remotePeer, msg.Payload, channel, message.ProtocolTypeUnicast) if err != nil { m.log. Error(). @@ -694,15 +687,12 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag return } } - - m.processAuthenticatedMessage(msg, decodedMsgPayload, remotePeer, network.ProtocolTypeUnicast) + m.processAuthenticatedMessage(msg, remotePeer, message.ProtocolTypeUnicast) } // processAuthenticatedMessage processes a message and a source (indicated by its peer ID) and eventually passes it to the overlay // In particular, it populates the `OriginID` field of the message with a Flow ID translated from this source. -// The assumption is that the message has been authenticated at the network level (libp2p) to originate from the peer with ID `peerID` -// this requirement is fulfilled by e.g. the output of readConnection and readSubscription -func (m *Middleware) processAuthenticatedMessage(msg *message.Message, decodedMsgPayload interface{}, peerID peer.ID, protocol network.ProtocolType) { +func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID peer.ID, protocol message.ProtocolType) { originId, err := m.idTranslator.GetFlowID(peerID) if err != nil { // this error should never happen. by the time the message gets here, the peer should be @@ -715,6 +705,35 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, decodedMs return } + channel := channels.Channel(msg.ChannelID) + decodedMsgPayload, err := m.codec.Decode(msg.Payload) + switch { + case codec.IsErrUnknownMsgCode(err): + // slash peer if message contains unknown message code byte + violation := &slashing.Violation{ + PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, + } + m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + return + case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): + // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte + violation := &slashing.Violation{ + PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, + } + m.slashingViolationsConsumer.OnInvalidMsgError(violation) + return + case err != nil: + // this condition should never happen and indicates there's a bug + // don't crash as a result of external inputs since that creates a DoS vector + // collect slashing data because this could potentially lead to slashing + err = fmt.Errorf("unexpected error during message validation: %w", err) + violation := &slashing.Violation{ + PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, + } + m.slashingViolationsConsumer.OnUnexpectedError(violation) + return + } + scope, err := network.NewIncomingScope(originId, protocol, msg, decodedMsgPayload) if err != nil { m.log.Error(). @@ -817,6 +836,21 @@ func unicastMaxMsgSize(messageType string) int { } } +// unicastMaxMsgSizeByCode returns the max permissible size for a unicast message code +func unicastMaxMsgSizeByCode(payload []byte) (int, error) { + msgCode, err := codec.MessageCodeFromPayload(payload) + if err != nil { + return 0, err + } + _, messageType, err := codec.InterfaceFromMessageCode(msgCode) + if err != nil { + return 0, err + } + + maxSize := unicastMaxMsgSize(messageType) + return maxSize, nil +} + // unicastMaxMsgDuration returns the max duration to allow for a unicast send to complete func (m *Middleware) unicastMaxMsgDuration(messageType string) time.Duration { switch messageType { diff --git a/network/p2p/middleware/readSubscription.go b/network/p2p/middleware/readSubscription.go index 9e74f6992d2..442fb152453 100644 --- a/network/p2p/middleware/readSubscription.go +++ b/network/p2p/middleware/readSubscription.go @@ -16,19 +16,19 @@ import ( "github.com/onflow/flow-go/utils/logging" ) -// readSubscriptionCB the callback called when a new message is received on the read subscription -type readSubscriptionCB func(msg *message.Message, decodedMsgPayload interface{}, peerID peer.ID) +// ReadSubscriptionCallBackFunction the callback called when a new message is received on the read subscription +type ReadSubscriptionCallBackFunction func(msg *message.Message, peerID peer.ID) // readSubscription reads the messages coming in on the subscription and calls the given callback until // the context of the subscription is cancelled. type readSubscription struct { log zerolog.Logger sub p2p.Subscription - callback readSubscriptionCB + callback ReadSubscriptionCallBackFunction } // newReadSubscription reads the messages coming in on the subscription -func newReadSubscription(sub p2p.Subscription, callback readSubscriptionCB, log zerolog.Logger) *readSubscription { +func newReadSubscription(sub p2p.Subscription, callback ReadSubscriptionCallBackFunction, log zerolog.Logger) *readSubscription { r := readSubscription{ log: log.With().Str("channel", sub.Topic()).Logger(), sub: sub, @@ -77,6 +77,6 @@ func (r *readSubscription) receiveLoop(ctx context.Context) { } // call the callback - r.callback(validatorData.Message, validatorData.DecodedMsgPayload, validatorData.From) + r.callback(validatorData.Message, validatorData.From) } } diff --git a/network/p2p/mock/node_block_list_consumer.go b/network/p2p/mock/node_block_list_consumer.go new file mode 100644 index 00000000000..dfb92edba97 --- /dev/null +++ b/network/p2p/mock/node_block_list_consumer.go @@ -0,0 +1,33 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mockp2p + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// NodeBlockListConsumer is an autogenerated mock type for the NodeBlockListConsumer type +type NodeBlockListConsumer struct { + mock.Mock +} + +// OnNodeBlockListUpdate provides a mock function with given fields: list +func (_m *NodeBlockListConsumer) OnNodeBlockListUpdate(list flow.IdentifierList) { + _m.Called(list) +} + +type mockConstructorTestingTNewNodeBlockListConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewNodeBlockListConsumer creates a new instance of NodeBlockListConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewNodeBlockListConsumer(t mockConstructorTestingTNewNodeBlockListConsumer) *NodeBlockListConsumer { + mock := &NodeBlockListConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/rate_limiter_consumer.go b/network/p2p/mock/rate_limiter_consumer.go new file mode 100644 index 00000000000..14050d81f91 --- /dev/null +++ b/network/p2p/mock/rate_limiter_consumer.go @@ -0,0 +1,34 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mockp2p + +import ( + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// RateLimiterConsumer is an autogenerated mock type for the RateLimiterConsumer type +type RateLimiterConsumer struct { + mock.Mock +} + +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *RateLimiterConsumer) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) +} + +type mockConstructorTestingTNewRateLimiterConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewRateLimiterConsumer creates a new instance of RateLimiterConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRateLimiterConsumer(t mockConstructorTestingTNewRateLimiterConsumer) *RateLimiterConsumer { + mock := &RateLimiterConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/mock/unicast_rate_limiter_distributor.go b/network/p2p/mock/unicast_rate_limiter_distributor.go new file mode 100644 index 00000000000..415b18d778e --- /dev/null +++ b/network/p2p/mock/unicast_rate_limiter_distributor.go @@ -0,0 +1,40 @@ +// Code generated by mockery v2.13.1. DO NOT EDIT. + +package mockp2p + +import ( + p2p "github.com/onflow/flow-go/network/p2p" + mock "github.com/stretchr/testify/mock" + + peer "github.com/libp2p/go-libp2p/core/peer" +) + +// UnicastRateLimiterDistributor is an autogenerated mock type for the UnicastRateLimiterDistributor type +type UnicastRateLimiterDistributor struct { + mock.Mock +} + +// AddConsumer provides a mock function with given fields: consumer +func (_m *UnicastRateLimiterDistributor) AddConsumer(consumer p2p.RateLimiterConsumer) { + _m.Called(consumer) +} + +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *UnicastRateLimiterDistributor) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) +} + +type mockConstructorTestingTNewUnicastRateLimiterDistributor interface { + mock.TestingT + Cleanup(func()) +} + +// NewUnicastRateLimiterDistributor creates a new instance of UnicastRateLimiterDistributor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewUnicastRateLimiterDistributor(t mockConstructorTestingTNewUnicastRateLimiterDistributor) *UnicastRateLimiterDistributor { + mock := &UnicastRateLimiterDistributor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/p2p/network.go b/network/p2p/network.go index 54a0d025018..b5bf83c8c11 100644 --- a/network/p2p/network.go +++ b/network/p2p/network.go @@ -6,8 +6,6 @@ import ( "sync" "time" - "github.com/onflow/flow-go/utils/logging" - "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" @@ -21,9 +19,11 @@ import ( "github.com/onflow/flow-go/network" netcache "github.com/onflow/flow-go/network/cache" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p/conduit" "github.com/onflow/flow-go/network/queue" _ "github.com/onflow/flow-go/utils/binstat" + "github.com/onflow/flow-go/utils/logging" ) const ( @@ -378,7 +378,7 @@ func (n *Network) UnicastOnChannel(channel channels.Channel, payload interface{} channel, payload, n.codec.Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) if err != nil { return fmt.Errorf("could not generate outgoing message scope for unicast: %w", err) } @@ -390,7 +390,7 @@ func (n *Network) UnicastOnChannel(channel channels.Channel, payload interface{} return fmt.Errorf("failed to send message to %x: %w", targetID, err) } - n.metrics.OutboundMessageSent(msg.Size(), msg.Channel().String(), network.ProtocolTypeUnicast.String(), msg.PayloadType()) + n.metrics.OutboundMessageSent(msg.Size(), msg.Channel().String(), message.ProtocolTypeUnicast.String(), msg.PayloadType()) return nil } @@ -442,27 +442,27 @@ func (n *Network) removeSelfFilter() flow.IdentifierFilter { } // sendOnChannel sends the message on channel to targets. -func (n *Network) sendOnChannel(channel channels.Channel, message interface{}, targetIDs []flow.Identifier) error { +func (n *Network) sendOnChannel(channel channels.Channel, msg interface{}, targetIDs []flow.Identifier) error { n.logger.Debug(). - Interface("message", message). + Interface("message", msg). Str("channel", channel.String()). Str("target_ids", fmt.Sprintf("%v", targetIDs)). Msg("sending new message on channel") // generate network message (encoding) based on list of recipients - msg, err := network.NewOutgoingScope(targetIDs, channel, message, n.codec.Encode, network.ProtocolTypePubSub) + scope, err := network.NewOutgoingScope(targetIDs, channel, msg, n.codec.Encode, message.ProtocolTypePubSub) if err != nil { return fmt.Errorf("failed to generate outgoing message scope %s: %w", channel, err) } // publish the message through the channel, however, the message // is only restricted to targetIDs (if they subscribed to channel). - err = n.mw.Publish(msg) + err = n.mw.Publish(scope) if err != nil { return fmt.Errorf("failed to send message on channel %s: %w", channel, err) } - n.metrics.OutboundMessageSent(msg.Size(), msg.Channel().String(), network.ProtocolTypePubSub.String(), msg.PayloadType()) + n.metrics.OutboundMessageSent(scope.Size(), scope.Channel().String(), message.ProtocolTypePubSub.String(), scope.PayloadType()) return nil } diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index a52289f72da..959bef7cb0a 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -70,9 +70,11 @@ func DefaultLibP2PNodeFactory(log zerolog.Logger, onInterceptPeerDialFilters, onInterceptSecuredFilters []p2p.PeerFilter, connectionPruning bool, updateInterval time.Duration, - rCfg *ResourceManagerConfig) LibP2PFactoryFunc { + rCfg *ResourceManagerConfig, + unicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor, +) LibP2PFactoryFunc { return func() (p2p.LibP2PNode, error) { - builder := DefaultNodeBuilder(log, + builder, err := DefaultNodeBuilder(log, address, flowKey, sporkId, @@ -85,7 +87,13 @@ func DefaultLibP2PNodeFactory(log zerolog.Logger, peerScoringEnabled, connectionPruning, updateInterval, - rCfg) + rCfg, + unicastRateLimiterDistributor) + + if err != nil { + return nil, fmt.Errorf("could not create node builder: %w", err) + } + return builder.Build() } } @@ -101,6 +109,7 @@ type NodeBuilder interface { EnableGossipSubPeerScoring(provider module.IdentityProvider, ops ...scoring.PeerScoreParamsOption) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder + SetRateLimiterDistributor(consumer p2p.UnicastRateLimiterDistributor) NodeBuilder Build() (p2p.LibP2PNode, error) } @@ -140,6 +149,7 @@ type LibP2PNodeBuilder struct { peerManagerUpdateInterval time.Duration peerScoringParameterOptions []scoring.PeerScoreParamsOption createNode CreateNodeFunc + rateLimiterDistributor p2p.UnicastRateLimiterDistributor } func NewNodeBuilder(logger zerolog.Logger, @@ -230,6 +240,11 @@ func (builder *LibP2PNodeBuilder) SetCreateNode(f CreateNodeFunc) NodeBuilder { return builder } +func (builder *LibP2PNodeBuilder) SetRateLimiterDistributor(distributor p2p.UnicastRateLimiterDistributor) NodeBuilder { + builder.rateLimiterDistributor = distributor + return builder +} + func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf GossipSubFactoryFunc, cf GossipSubAdapterConfigFunc) NodeBuilder { builder.gossipSubFactory = gf builder.gossipSubConfigFunc = cf @@ -272,10 +287,18 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { return nil, fmt.Errorf("could not get allowed file descriptors: %w", err) } - mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits.Scale(mem, fd)), rcmgr.WithMetrics(builder.metrics)) + l := limits.Scale(mem, fd) + mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(l), rcmgr.WithMetrics(builder.metrics)) if err != nil { return nil, fmt.Errorf("could not create libp2p resource manager: %w", err) } + builder.logger.Info(). + Str("key", keyResourceManagerLimit). + Int64("allowed_memory", mem). + Int("allowed_file_descriptors", fd). + Msg("allowed memory and file descriptors are fetched from the system") + newLimitConfigLogger(builder.logger).logResourceManagerLimits(l) + opts = append(opts, libp2p.ResourceManager(mgr)) builder.logger.Info().Msg("libp2p resource manager is set to default with metrics") } @@ -309,6 +332,10 @@ func (builder *LibP2PNodeBuilder) Build() (p2p.LibP2PNode, error) { } peerManager = connection.NewPeerManager(builder.logger, builder.peerManagerUpdateInterval, connector) + + if builder.rateLimiterDistributor != nil { + builder.rateLimiterDistributor.AddConsumer(peerManager) + } } node := builder.createNode(builder.logger, h, pCache, unicastManager, peerManager) @@ -451,14 +478,20 @@ func DefaultNodeBuilder(log zerolog.Logger, peerScoringEnabled bool, connectionPruning bool, updateInterval time.Duration, - rCfg *ResourceManagerConfig) NodeBuilder { - connManager := connection.NewConnManager(log, metrics) + rCfg *ResourceManagerConfig, + unicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor) (NodeBuilder, error) { + + connManager, err := connection.NewConnManager(log, metrics, connection.DefaultConnManagerConfig()) + if err != nil { + return nil, fmt.Errorf("could not create connection manager: %w", err) + } // set the default connection gater peer filters for both InterceptPeerDial and InterceptSecured callbacks peerFilter := notEjectedPeerFilter(idProvider) peerFilters := []p2p.PeerFilter{peerFilter} connGater := connection.NewConnGater(log, + idProvider, connection.WithOnInterceptPeerDialFilters(append(peerFilters, onInterceptPeerDialFilters...)), connection.WithOnInterceptSecuredFilters(append(peerFilters, onInterceptSecuredFilters...))) @@ -470,7 +503,8 @@ func DefaultNodeBuilder(log zerolog.Logger, return dht.NewDHT(ctx, host, unicast.FlowDHTProtocolID(sporkId), log, metrics, dht.AsServer()) }). SetPeerManagerOptions(connectionPruning, updateInterval). - SetCreateNode(DefaultCreateNodeFunc) + SetCreateNode(DefaultCreateNodeFunc). + SetRateLimiterDistributor(unicastRateLimiterDistributor) if peerScoringEnabled { builder.EnableGossipSubPeerScoring(idProvider) @@ -481,5 +515,5 @@ func DefaultNodeBuilder(log zerolog.Logger, builder.SetSubscriptionFilter(subscription.NewRoleBasedFilter(r, idProvider)) } - return builder + return builder, nil } diff --git a/network/p2p/p2pbuilder/utils.go b/network/p2p/p2pbuilder/utils.go index 0ce87132712..29b4d143698 100644 --- a/network/p2p/p2pbuilder/utils.go +++ b/network/p2p/p2pbuilder/utils.go @@ -4,11 +4,16 @@ import ( "fmt" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" + "github.com/rs/zerolog" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/network/p2p" ) +const keyResourceManagerLimit = "libp2p_resource_manager_limit" + // notEjectedPeerFilter returns a PeerFilter that will return an error if the peer is unknown or ejected. func notEjectedPeerFilter(idProvider module.IdentityProvider) p2p.PeerFilter { return func(p peer.ID) error { @@ -21,3 +26,97 @@ func notEjectedPeerFilter(idProvider module.IdentityProvider) p2p.PeerFilter { return nil } } + +type limitConfigLogger struct { + logger zerolog.Logger +} + +// newLimitConfigLogger creates a new limitConfigLogger. +func newLimitConfigLogger(logger zerolog.Logger) *limitConfigLogger { + return &limitConfigLogger{logger: logger} +} + +// withBaseLimit appends the base limit to the logger with the given prefix. +func (l *limitConfigLogger) withBaseLimit(prefix string, baseLimit rcmgr.BaseLimit) zerolog.Logger { + return l.logger.With(). + Str("key", keyResourceManagerLimit). + Int(fmt.Sprintf("%s_streams", prefix), baseLimit.Streams). + Int(fmt.Sprintf("%s_streams_inbound", prefix), baseLimit.StreamsInbound). + Int(fmt.Sprintf("%s_streams_outbound", prefix), baseLimit.StreamsOutbound). + Int(fmt.Sprintf("%s_conns", prefix), baseLimit.Conns). + Int(fmt.Sprintf("%s_conns_inbound", prefix), baseLimit.ConnsInbound). + Int(fmt.Sprintf("%s_conns_outbound", prefix), baseLimit.ConnsOutbound). + Int(fmt.Sprintf("%s_file_descriptors", prefix), baseLimit.FD). + Int64(fmt.Sprintf("%s_memory", prefix), baseLimit.Memory).Logger() +} + +func (l *limitConfigLogger) logResourceManagerLimits(config rcmgr.LimitConfig) { + l.logGlobalResourceLimits(config) + l.logServiceLimits(config.Service) + l.logProtocolLimits(config.Protocol) + l.logPeerLimits(config.Peer) + l.logPeerProtocolLimits(config.ProtocolPeer) +} + +func (l *limitConfigLogger) logGlobalResourceLimits(config rcmgr.LimitConfig) { + lg := l.withBaseLimit("system", config.System) + lg.Info().Msg("system limits set") + + lg = l.withBaseLimit("transient", config.Transient) + lg.Info().Msg("transient limits set") + + lg = l.withBaseLimit("allowed_listed_system", config.AllowlistedSystem) + lg.Info().Msg("allowed listed system limits set") + + lg = l.withBaseLimit("allowed_lister_transient", config.AllowlistedTransient) + lg.Info().Msg("allowed listed transient limits set") + + lg = l.withBaseLimit("service_default", config.ServiceDefault) + lg.Info().Msg("service default limits set") + + lg = l.withBaseLimit("service_peer_default", config.ServicePeerDefault) + lg.Info().Msg("service peer default limits set") + + lg = l.withBaseLimit("protocol_default", config.ProtocolDefault) + lg.Info().Msg("protocol default limits set") + + lg = l.withBaseLimit("protocol_peer_default", config.ProtocolPeerDefault) + lg.Info().Msg("protocol peer default limits set") + + lg = l.withBaseLimit("peer_default", config.PeerDefault) + lg.Info().Msg("peer default limits set") + + lg = l.withBaseLimit("connections", config.Conn) + lg.Info().Msg("connection limits set") + + lg = l.withBaseLimit("streams", config.Stream) + lg.Info().Msg("stream limits set") +} + +func (l *limitConfigLogger) logServiceLimits(s map[string]rcmgr.BaseLimit) { + for sName, sLimits := range s { + lg := l.withBaseLimit(fmt.Sprintf("service_%s", sName), sLimits) + lg.Info().Msg("service limits set") + } +} + +func (l *limitConfigLogger) logProtocolLimits(p map[protocol.ID]rcmgr.BaseLimit) { + for pName, pLimits := range p { + lg := l.withBaseLimit(fmt.Sprintf("protocol_%s", pName), pLimits) + lg.Info().Msg("protocol limits set") + } +} + +func (l *limitConfigLogger) logPeerLimits(p map[peer.ID]rcmgr.BaseLimit) { + for pId, pLimits := range p { + lg := l.withBaseLimit(fmt.Sprintf("peer_%s", pId.String()), pLimits) + lg.Info().Msg("peer limits set") + } +} + +func (l *limitConfigLogger) logPeerProtocolLimits(p map[protocol.ID]rcmgr.BaseLimit) { + for pName, pLimits := range p { + lg := l.withBaseLimit(fmt.Sprintf("protocol_peer_%s", pName), pLimits) + lg.Info().Msg("protocol peer limits set") + } +} diff --git a/network/p2p/p2pnode/libp2pNode.go b/network/p2p/p2pnode/libp2pNode.go index e0d0a4ad8c8..311b532cc31 100644 --- a/network/p2p/p2pnode/libp2pNode.go +++ b/network/p2p/p2pnode/libp2pNode.go @@ -27,6 +27,7 @@ import ( "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/unicast" + "github.com/onflow/flow-go/utils/logging" ) const ( @@ -150,6 +151,13 @@ func (n *Node) RemovePeer(peerID peer.ID) error { if err != nil { return fmt.Errorf("failed to remove peer %s: %w", peerID, err) } + // logging with suspicious level as we only expect to disconnect from a peer if it is not part of the + // protocol state. + n.logger.Warn(). + Str("peer_id", peerID.String()). + Bool(logging.KeySuspicious, true). + Msg("disconnected from peer") + return nil } diff --git a/network/p2p/p2pnode/libp2pNode_test.go b/network/p2p/p2pnode/libp2pNode_test.go index abd0c13a760..7bc8e854e21 100644 --- a/network/p2p/p2pnode/libp2pNode_test.go +++ b/network/p2p/p2pnode/libp2pNode_test.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/mock" + mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/p2putils" @@ -155,18 +155,20 @@ func TestConnGater(t *testing.T) { signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) sporkID := unittest.IdentifierFixture() + idProvider := mockmodule.NewIdentityProvider(t) node1Peers := unittest.NewProtectedMap[peer.ID, struct{}]() node1, identity1 := p2ptest.NodeFixture( t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(func(pid peer.ID) error { + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node1Peers.Has(pid) { return fmt.Errorf("peer id not found: %s", pid.String()) } return nil }))) + idProvider.On("ByPeerID", node1.Host().ID()).Return(&identity1, true).Maybe() p2ptest.StartNode(t, signalerCtx, node1, 100*time.Millisecond) defer p2ptest.StopNode(t, node1, cancel, 100*time.Millisecond) @@ -178,12 +180,16 @@ func TestConnGater(t *testing.T) { node2, identity2 := p2ptest.NodeFixture( t, sporkID, t.Name(), - p2ptest.WithConnectionGater(testutils.NewConnectionGater(func(pid peer.ID) error { + p2ptest.WithConnectionGater(testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { if !node2Peers.Has(pid) { return fmt.Errorf("id not found: %s", pid.String()) } return nil }))) + idProvider.On("ByPeerID", node2.Host().ID()).Return(&identity2, + + true).Maybe() + p2ptest.StartNode(t, signalerCtx, node2, 100*time.Millisecond) defer p2ptest.StopNode(t, node2, cancel, 100*time.Millisecond) @@ -220,9 +226,8 @@ func TestNode_HasSubscription(t *testing.T) { defer p2ptest.StopNode(t, node, cancel, 100*time.Millisecond) logger := unittest.Logger() - met := mock.NewNetworkMetrics(t) - topicValidator := validator.TopicValidator(logger, unittest.NetworkCodec(), unittest.NetworkSlashingViolationsConsumer(logger, met), func(id peer.ID) error { + topicValidator := validator.TopicValidator(logger, func(id peer.ID) error { return nil }) diff --git a/network/p2p/rate_limiter.go b/network/p2p/rate_limiter.go index b28892070e3..98487ad7197 100644 --- a/network/p2p/rate_limiter.go +++ b/network/p2p/rate_limiter.go @@ -36,3 +36,17 @@ func WithGetTimeNowFunc(now GetTimeNow) RateLimiterOpt { limiter.SetTimeNowFunc(now) } } + +// UnicastRateLimiterDistributor consumes then distributes notifications from the ratelimit.RateLimiters whenever a peer is rate limited. +type UnicastRateLimiterDistributor interface { + RateLimiterConsumer + AddConsumer(consumer RateLimiterConsumer) +} + +// RateLimiterConsumer consumes notifications from the ratelimit.RateLimiters whenever a peer is rate limited. +// Implementations must: +// - be concurrency safe +// - be non-blocking +type RateLimiterConsumer interface { + OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) +} diff --git a/network/p2p/scoring/app_score_test.go b/network/p2p/scoring/app_score_test.go index 11fba04213f..380cbb9544d 100644 --- a/network/p2p/scoring/app_score_test.go +++ b/network/p2p/scoring/app_score_test.go @@ -12,7 +12,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" @@ -59,7 +58,7 @@ func TestFullGossipSubConnectivity(t *testing.T) { defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector()) + logger := unittest.Logger() // all nodes subscribe to block topic (common topic among all roles) @@ -67,19 +66,19 @@ func TestFullGossipSubConnectivity(t *testing.T) { groupOneSubs := make([]p2p.Subscription, len(groupOneNodes)) var err error for i, node := range groupOneNodes { - groupOneSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + groupOneSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) } // group two groupTwoSubs := make([]p2p.Subscription, len(groupTwoNodes)) for i, node := range groupTwoNodes { - groupTwoSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + groupTwoSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) } // access node group accessNodeSubs := make([]p2p.Subscription, len(accessNodeGroup)) for i, node := range accessNodeGroup { - accessNodeSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + accessNodeSubs[i], err = node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) } @@ -180,20 +179,20 @@ func testGossipSubMessageDeliveryUnderNetworkPartition(t *testing.T, honestPeerS defer p2ptest.StopNodes(t, allNodes, cancel, 2*time.Second) blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector()) + logger := unittest.Logger() // all nodes subscribe to block topic (common topic among all roles) - _, err := con1Node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err := con1Node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) - con2Sub, err := con2Node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + con2Sub, err := con2Node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // access node group accessNodeSubs := make([]p2p.Subscription, len(accessNodeGroup)) for i, node := range accessNodeGroup { - sub, err := node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub, err := node.Subscribe(blockTopic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) accessNodeSubs[i] = sub } diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index f2b8fbaf39c..d0341be891f 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -19,7 +19,6 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" @@ -209,9 +208,8 @@ func TestSubscriptionValidator_Integration(t *testing.T) { defer p2ptest.StopNodes(t, nodes, cancel, 100*time.Millisecond) blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector()) - topicValidator := flowpubsub.TopicValidator(unittest.Logger(), unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter()) + topicValidator := flowpubsub.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter()) // wait for the subscriptions to be established p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) diff --git a/network/p2p/subscription/subscription_filter_test.go b/network/p2p/subscription/subscription_filter_test.go index bf2e73ff65a..4173e13a431 100644 --- a/network/p2p/subscription/subscription_filter_test.go +++ b/network/p2p/subscription/subscription_filter_test.go @@ -14,7 +14,6 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/p2p" @@ -48,7 +47,7 @@ func TestFilterSubscribe(t *testing.T) { badTopic := channels.TopicFromChannel(channels.SyncCommittee, sporkId) logger := unittest.Logger() - topicValidator := flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()), unittest.AllowAllPeerFilter()) + topicValidator := flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter()) sub1, err := node1.Subscribe(badTopic, topicValidator) require.NoError(t, err) @@ -122,7 +121,7 @@ func TestCanSubscribe(t *testing.T) { defer p2ptest.StopNode(t, collectionNode, cancel, 1*time.Second) logger := unittest.Logger() - topicValidator := flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()), unittest.AllowAllPeerFilter()) + topicValidator := flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter()) goodTopic := channels.TopicFromChannel(channels.ProvideCollections, sporkId) _, err := collectionNode.Subscribe(goodTopic, topicValidator) diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 6cdd751f750..f17d9850f6d 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -54,12 +54,14 @@ func NodeFixture( ) (p2p.LibP2PNode, flow.Identity) { // default parameters parameters := &NodeFixtureParameters{ - HandlerFunc: func(network.Stream) {}, - Unicasts: nil, - Key: NetworkingKeyFixtures(t), - Address: unittest.DefaultAddress, - Logger: unittest.Logger().Level(zerolog.ErrorLevel), - Role: flow.RoleCollection, + HandlerFunc: func(network.Stream) {}, + Unicasts: nil, + Key: NetworkingKeyFixtures(t), + Address: unittest.DefaultAddress, + Logger: unittest.Logger().Level(zerolog.ErrorLevel), + Role: flow.RoleCollection, + Metrics: metrics.NewNoopCollector(), + ResourceManager: testutils.NewResourceManager(t), } for _, opt := range opts { @@ -73,13 +75,12 @@ func NodeFixture( logger := parameters.Logger.With().Hex("node_id", logging.ID(identity.NodeID)).Logger() - noopMetrics := metrics.NewNoopCollector() - connManager := connection.NewConnManager(logger, noopMetrics) - resourceManager := testutils.NewResourceManager(t) + connManager, err := connection.NewConnManager(logger, parameters.Metrics, connection.DefaultConnManagerConfig()) + require.NoError(t, err) builder := p2pbuilder.NewNodeBuilder( logger, - metrics.NewNoopCollector(), + parameters.Metrics, parameters.Address, parameters.Key, sporkID, @@ -89,13 +90,16 @@ func NodeFixture( return p2pdht.NewDHT(c, h, protocol.ID(unicast.FlowDHTProtocolIDPrefix+sporkID.String()+"/"+dhtPrefix), logger, - noopMetrics, + parameters.Metrics, parameters.DhtOptions..., ) }). - SetResourceManager(resourceManager). SetCreateNode(p2pbuilder.DefaultCreateNodeFunc) + if parameters.ResourceManager != nil { + builder.SetResourceManager(parameters.ResourceManager) + } + if parameters.ConnGater != nil { builder.SetConnectionGater(parameters.ConnGater) } @@ -117,6 +121,10 @@ func NodeFixture( builder.SetGossipSubFactory(parameters.GossipSubFactory, parameters.GossipSubConfig) } + if parameters.ConnManager != nil { + builder.SetConnectionManager(parameters.ConnManager) + } + n, err := builder.Build() require.NoError(t, err) @@ -151,8 +159,11 @@ type NodeFixtureParameters struct { UpdateInterval time.Duration // peer manager parameter PeerProvider p2p.PeersProvider // peer manager parameter ConnGater connmgr.ConnectionGater + ConnManager connmgr.ConnManager GossipSubFactory p2pbuilder.GossipSubFactoryFunc GossipSubConfig p2pbuilder.GossipSubAdapterConfigFunc + Metrics module.LibP2PMetrics + ResourceManager network.ResourceManager } func WithPeerScoringEnabled(idProvider module.IdentityProvider) NodeFixtureParameterOption { @@ -206,6 +217,12 @@ func WithConnectionGater(connGater connmgr.ConnectionGater) NodeFixtureParameter } } +func WithConnectionManager(connManager connmgr.ConnManager) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.ConnManager = connManager + } +} + func WithRole(role flow.Role) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.Role = role @@ -224,6 +241,20 @@ func WithLogger(logger zerolog.Logger) NodeFixtureParameterOption { } } +func WithMetricsCollector(metrics module.LibP2PMetrics) NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.Metrics = metrics + } +} + +// WithDefaultResourceManager sets the resource manager to nil, which will cause the node to use the default resource manager. +// Otherwise, it uses the resource manager provided by the test (the infinite resource manager). +func WithDefaultResourceManager() NodeFixtureParameterOption { + return func(p *NodeFixtureParameters) { + p.ResourceManager = nil + } +} + // NodesFixture is a test fixture that creates a number of libp2p nodes with the given callback function for stream handling. // It returns the nodes and their identities. func NodesFixture(t *testing.T, sporkID flow.Identifier, dhtPrefix string, count int, opts ...NodeFixtureParameterOption) ([]p2p.LibP2PNode, @@ -343,14 +374,11 @@ func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p. _, topic := messageFactory() subs := make([]p2p.Subscription, len(nodes)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector()) for i, node := range nodes { ps, err := node.Subscribe( topic, validator.TopicValidator( unittest.Logger(), - unittest.NetworkCodec(), - slashingViolationsConsumer, unittest.AllowAllPeerFilter())) require.NoError(t, err) subs[i] = ps diff --git a/network/p2p/test/sporking_test.go b/network/p2p/test/sporking_test.go index bc9dc09c4fc..1fa099013f3 100644 --- a/network/p2p/test/sporking_test.go +++ b/network/p2p/test/sporking_test.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/flow-go/model/flow" libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/message" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" @@ -20,7 +21,6 @@ import ( "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" @@ -207,7 +207,7 @@ func TestOneToKCrosstalkPrevention(t *testing.T) { topicBeforeSpork := channels.TopicFromChannel(channels.TestNetworkChannel, previousSporkId) logger := unittest.Logger() - topicValidator := flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()), unittest.AllowAllPeerFilter()) + topicValidator := flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter()) // both nodes are initially on the same spork and subscribed to the same topic _, err = node1.Subscribe(topicBeforeSpork, topicValidator) @@ -275,7 +275,7 @@ func testOneToKMessagingSucceeds(ctx context.Context, Text: string("hello"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypePubSub) + message.ProtocolTypePubSub) require.NoError(t, err) sentData, err := sentMsg.Proto().Marshal() @@ -308,7 +308,7 @@ func testOneToKMessagingFails(ctx context.Context, Text: string("hello"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypePubSub) + message.ProtocolTypePubSub) require.NoError(t, err) sentData, err := sentMsg.Proto().Marshal() diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index c7418cf54a3..18229bd2e81 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -76,14 +76,12 @@ func TestTopicValidator_Unstaked(t *testing.T) { // sn1 <-> sn2 require.NoError(t, sn1.AddPeer(ctx, pInfo2)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) - // sn1 will subscribe with is staked callback that should force the TopicValidator to drop the message received from sn2 - sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, isStaked)) + sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, isStaked)) require.NoError(t, err) // sn2 will subscribe with an unauthenticated callback to allow it to send the unauthenticated message - _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -132,11 +130,10 @@ func TestTopicValidator_PublicChannel(t *testing.T) { // sn1 <-> sn2 require.NoError(t, sn1.AddPeer(ctx, pInfo2)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) // sn1 & sn2 will subscribe with unauthenticated callback to allow it to send and receive unauthenticated messages - sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) - sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -192,14 +189,12 @@ func TestTopicValidator_TopicMismatch(t *testing.T) { // sn1 <-> sn2 require.NoError(t, sn1.AddPeer(ctx, pInfo2)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) - // sn2 will subscribe with an unauthenticated callback to allow processing of message after the authorization check - _, err = sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // sn2 will subscribe with an unauthenticated callback to allow it to send the unauthenticated message - _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -245,14 +240,12 @@ func TestTopicValidator_InvalidTopic(t *testing.T) { // sn1 <-> sn2 require.NoError(t, sn1.AddPeer(ctx, pInfo2)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) - // sn2 will subscribe with an unauthenticated callback to allow processing of message after the authorization check - _, err = sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // sn2 will subscribe with an unauthenticated callback to allow it to send the unauthenticated message - _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -321,13 +314,12 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { require.NoError(t, sn1.AddPeer(ctx, pInfo2)) require.NoError(t, an1.AddPeer(ctx, pInfo1)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) // sn1 and sn2 subscribe to the topic with the topic validator - sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - sub3, err := an1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub3, err := an1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -426,11 +418,10 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { // sn1 <-> sn2 require.NoError(t, sn1.AddPeer(ctx, pInfo2)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) // sn1 subscribe to the topic with the topic validator, while sn2 will subscribe without the topic validator to allow sn2 to publish unauthorized messages - sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + _, err = sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -502,13 +493,12 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { require.NoError(t, sn1.AddPeer(ctx, pInfo2)) require.NoError(t, an1.AddPeer(ctx, pInfo1)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) // sn1 subscribe to the topic with the topic validator, while sn2 will subscribe without the topic validator to allow sn2 to publish unauthorized messages - sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub1, err := sn1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub2, err := sn2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) - sub3, err := an1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter())) + sub3, err := an1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter())) require.NoError(t, err) // let nodes form the mesh @@ -595,12 +585,11 @@ func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { require.NoError(t, ln1.AddPeer(ctx, pInfo2)) require.NoError(t, ln3.AddPeer(ctx, pInfo1)) - slashingViolationsConsumer := unittest.NetworkSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) - sub1, err := ln1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub1, err := ln1.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - sub2, err := ln2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub2, err := ln2.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) - sub3, err := ln3.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.NetworkCodec(), slashingViolationsConsumer, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) + sub3, err := ln3.Subscribe(topic, flowpubsub.TopicValidator(logger, unittest.AllowAllPeerFilter(), pubsubMessageValidator)) require.NoError(t, err) // let nodes form the mesh diff --git a/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go b/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go index c79c0edf53f..16df3b62f78 100644 --- a/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go +++ b/network/p2p/unicast/ratelimit/bandwidth_rate_limiter_test.go @@ -7,6 +7,7 @@ import ( "github.com/onflow/flow-go/model/flow" libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/message" "github.com/stretchr/testify/require" "golang.org/x/time/rate" @@ -43,7 +44,7 @@ func TestBandWidthRateLimiter_Allow(t *testing.T) { Text: string(b), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(t, err) allowed := bandwidthRateLimiter.Allow(peerID, msg.Size()) @@ -96,7 +97,7 @@ func TestBandWidthRateLimiter_IsRateLimited(t *testing.T) { Text: string(b), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(t, err) allowed := bandwidthRateLimiter.Allow(peerID, msg.Size()) diff --git a/network/p2p/unicast/ratelimit/distributor.go b/network/p2p/unicast/ratelimit/distributor.go new file mode 100644 index 00000000000..f91ea2d0810 --- /dev/null +++ b/network/p2p/unicast/ratelimit/distributor.go @@ -0,0 +1,40 @@ +package ratelimit + +import ( + "sync" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/onflow/flow-go/network/p2p" +) + +// UnicastRateLimiterDistributor subscribes to rate limited peer events from RateLimiters. +type UnicastRateLimiterDistributor struct { + consumers []p2p.RateLimiterConsumer + lock sync.RWMutex +} + +var _ p2p.RateLimiterConsumer = (*UnicastRateLimiterDistributor)(nil) + +// NewUnicastRateLimiterDistributor returns a new UnicastRateLimiterDistributor. +func NewUnicastRateLimiterDistributor() *UnicastRateLimiterDistributor { + return &UnicastRateLimiterDistributor{ + consumers: make([]p2p.RateLimiterConsumer, 0), + } +} + +// AddConsumer adds a consumer to the consumers list. +func (r *UnicastRateLimiterDistributor) AddConsumer(consumer p2p.RateLimiterConsumer) { + r.lock.Lock() + defer r.lock.Unlock() + r.consumers = append(r.consumers, consumer) +} + +// OnRateLimitedPeer invokes each consumer callback with the rate limited peer info. +func (r *UnicastRateLimiterDistributor) OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) { + r.lock.RLock() + defer r.lock.RUnlock() + for _, consumer := range r.consumers { + consumer.OnRateLimitedPeer(pid, role, msgType, topic, reason) + } +} diff --git a/network/p2p/unicast/ratelimit/noop_rate_limiter.go b/network/p2p/unicast/ratelimit/noop_rate_limiter.go index 4257f9b4a6e..87cba2b743a 100644 --- a/network/p2p/unicast/ratelimit/noop_rate_limiter.go +++ b/network/p2p/unicast/ratelimit/noop_rate_limiter.go @@ -31,7 +31,7 @@ func NoopRateLimiters() *RateLimiters { return &RateLimiters{ MessageRateLimiter: &NoopRateLimiter{}, BandWidthRateLimiter: &NoopRateLimiter{}, - OnRateLimitedPeer: nil, disabled: true, + notifier: NewUnicastRateLimiterDistributor(), } } diff --git a/network/p2p/unicast/ratelimit/rate_limiters.go b/network/p2p/unicast/ratelimit/rate_limiters.go index 6428a371464..08a1c7bb2ad 100644 --- a/network/p2p/unicast/ratelimit/rate_limiters.go +++ b/network/p2p/unicast/ratelimit/rate_limiters.go @@ -36,22 +36,35 @@ func WithDisabledRateLimiting(disabled bool) RateLimitersOption { } } +func WithMessageRateLimiter(messageLimiter p2p.RateLimiter) RateLimitersOption { + return func(r *RateLimiters) { + r.MessageRateLimiter = messageLimiter + } +} + +func WithBandwidthRateLimiter(bandwidthLimiter p2p.RateLimiter) RateLimitersOption { + return func(r *RateLimiters) { + r.BandWidthRateLimiter = bandwidthLimiter + } +} + +func WithNotifier(notifier p2p.RateLimiterConsumer) RateLimitersOption { + return func(r *RateLimiters) { + r.notifier = notifier + } +} + // RateLimiters used to manage stream and bandwidth rate limiters type RateLimiters struct { MessageRateLimiter p2p.RateLimiter BandWidthRateLimiter p2p.RateLimiter - OnRateLimitedPeer OnRateLimitedPeerFunc // the callback called each time a peer is rate limited - disabled bool // flag allows rate limiter to collect metrics without rate limiting if set to false + notifier p2p.RateLimiterConsumer + disabled bool // flag allows rate limiter to collect metrics without rate limiting if set to false } // NewRateLimiters returns *RateLimiters -func NewRateLimiters(messageLimiter, bandwidthLimiter p2p.RateLimiter, onRateLimitedPeer OnRateLimitedPeerFunc, opts ...RateLimitersOption) *RateLimiters { - r := &RateLimiters{ - MessageRateLimiter: messageLimiter, - BandWidthRateLimiter: bandwidthLimiter, - OnRateLimitedPeer: onRateLimitedPeer, - disabled: true, - } +func NewRateLimiters(opts ...RateLimitersOption) *RateLimiters { + r := NoopRateLimiters() for _, opt := range opts { opt(r) @@ -68,7 +81,7 @@ func (r *RateLimiters) MessageAllowed(peerID peer.ID) bool { } if !r.MessageRateLimiter.Allow(peerID, 0) { // 0 is not used for message rate limiter. It is only used for bandwidth rate limiter. - r.onRateLimitedPeer(peerID, "", "", "", ReasonMessageCount) + r.notifier.OnRateLimitedPeer(peerID, "", "", "", ReasonMessageCount.String()) // avoid rate limiting during dry run return r.disabled @@ -85,7 +98,7 @@ func (r *RateLimiters) BandwidthAllowed(peerID peer.ID, originRole string, msgSi } if !r.BandWidthRateLimiter.Allow(peerID, msgSize) { - r.onRateLimitedPeer(peerID, originRole, msgType, msgTopic, ReasonBandwidth) + r.notifier.OnRateLimitedPeer(peerID, originRole, msgType, msgTopic.String(), ReasonBandwidth.String()) // avoid rate limiting during dry runs if disabled set to false return r.disabled @@ -94,13 +107,6 @@ func (r *RateLimiters) BandwidthAllowed(peerID peer.ID, originRole string, msgSi return true } -// onRateLimitedPeer invokes the r.onRateLimitedPeer callback if it is not nil -func (r *RateLimiters) onRateLimitedPeer(peerID peer.ID, role string, msgType string, topic channels.Topic, reason RateLimitReason) { - if r.OnRateLimitedPeer != nil { - r.OnRateLimitedPeer(peerID, role, msgType, topic, reason) - } -} - // Start starts the cleanup loop for all limiters func (r *RateLimiters) Start() { if r.MessageRateLimiter != nil { diff --git a/network/slashing/violations_consumer.go b/network/slashing/violations_consumer.go index cc6850743ed..cf1f8ea7d85 100644 --- a/network/slashing/violations_consumer.go +++ b/network/slashing/violations_consumer.go @@ -2,7 +2,7 @@ package slashing import ( "github.com/onflow/flow-go/model/flow" - network "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/message" ) @@ -30,8 +30,9 @@ type ViolationsConsumer interface { type Violation struct { Identity *flow.Identity PeerID string + OriginID flow.Identifier MsgType string - Channel network.Channel - Protocol message.Protocol + Channel channels.Channel + Protocol message.ProtocolType Err error } diff --git a/network/test/middleware_test.go b/network/test/middleware_test.go index fc29cbcf54b..8d9d281b9c6 100644 --- a/network/test/middleware_test.go +++ b/network/test/middleware_test.go @@ -24,15 +24,17 @@ import ( libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/module/observable" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/middleware" "github.com/onflow/flow-go/network/p2p/p2pnode" + p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" @@ -195,7 +197,7 @@ func (m *MiddlewareTestSuite) TestUpdateNodeAddresses() { Text: "TestUpdateNodeAddresses", }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // message should fail to send because no address is known yet // for the new identity @@ -214,52 +216,53 @@ func (m *MiddlewareTestSuite) TestUpdateNodeAddresses() { } func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { - unittest.SkipUnless(m.T(), unittest.TEST_FLAKY, "disabling so that flaky metrics can be gathered before re-enabling") - // limiter limit will be set to 5 events/sec the 6th event per interval will be rate limited limit := rate.Limit(5) // burst per interval burst := 5 - messageRateLimiter := ratelimit.NewMessageRateLimiter(limit, burst, 1) + messageRateLimiter := ratelimit.NewMessageRateLimiter(limit, burst, 3) - // the onUnicastRateLimitedPeerFunc call back we will use to keep track of how many times a rate limit happens - // after 5 rate limits we will close ch. O - ch := make(chan struct{}) + // we only expect messages from the first middleware on the test suite + expectedPID, err := unittest.PeerIDFromFlowID(m.ids[0]) + require.NoError(m.T(), err) + + // the onRateLimit call back we will use to keep track of how many times a rate limit happens. rateLimits := atomic.NewUint64(0) - onRateLimit := func(peerID peer.ID, role, msgType string, topic channels.Topic, reason ratelimit.RateLimitReason) { - require.Equal(m.T(), reason, ratelimit.ReasonMessageCount) - // we only expect messages from the first middleware on the test suite - expectedPID, err := unittest.PeerIDFromFlowID(m.ids[0]) - require.NoError(m.T(), err) + onRateLimit := func(peerID peer.ID, role, msgType, topic, reason string) { + require.Equal(m.T(), reason, ratelimit.ReasonMessageCount.String()) require.Equal(m.T(), expectedPID, peerID) - // update hook calls rateLimits.Inc() } - rateLimiters := ratelimit.NewRateLimiters(messageRateLimiter, - &ratelimit.NoopRateLimiter{}, - onRateLimit, - ratelimit.WithDisabledRateLimiting(false)) + // setup rate limit distributor that will be used to track the number of rate limits via the onRateLimit callback. + consumer := testutils.NewRateLimiterConsumer(onRateLimit) + distributor := ratelimit.NewUnicastRateLimiterDistributor() + distributor.AddConsumer(consumer) - // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), m.logger, 1) + opts := []ratelimit.RateLimitersOption{ratelimit.WithMessageRateLimiter(messageRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} + rateLimiters := ratelimit.NewRateLimiters(opts...) - // create middleware - netmet := mock.NewNetworkMetrics(m.T()) - calls := 0 - netmet.On("InboundMessageReceived", mockery.Anything, mockery.Anything, mockery.Anything).Times(5).Run(func(args mockery.Arguments) { - calls++ - if calls == 5 { - close(ch) + idProvider := testutils.NewUpdatableIDProvider(m.ids) + // create a new staked identity + connGater := testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + if messageRateLimiter.IsRateLimited(pid) { + return fmt.Errorf("rate-limited peer") } + + return nil }) - // we expect 5 messages to be processed the rest will be rate limited - defer netmet.AssertNumberOfCalls(m.T(), "InboundMessageReceived", 5) + ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), + m.logger, + 1, + testutils.WithUnicastRateLimiterDistributor(distributor), + testutils.WithConnectionGater(connGater)) + idProvider.SetIdentities(append(m.ids, ids...)) + // create middleware mws, providers := testutils.GenerateMiddlewares(m.T(), m.logger, ids, @@ -267,16 +270,27 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { unittest.NetworkCodec(), m.slashingViolationsConsumer, testutils.WithUnicastRateLimiters(rateLimiters), - testutils.WithNetworkMetrics(netmet)) + testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(messageRateLimiter))) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) newId := ids[0] newMw := mws[0] + idList := flow.IdentityList(append(m.ids, newId)) + + providers[0].SetIdentities(idList) overlay := m.createOverlay(providers[0]) - overlay.On("Receive", m.ids[0].NodeID, mockery.AnythingOfType("*message.Message")).Return(nil) + + calls := atomic.NewUint64(0) + ch := make(chan struct{}) + overlay.On("Receive", mockery.AnythingOfType("*network.IncomingMessageScope")).Return(nil).Run(func(args mockery.Arguments) { + calls.Inc() + if calls.Load() >= 5 { + close(ch) + } + }) newMw.SetOverlay(overlay) @@ -291,16 +305,25 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { require.NoError(m.T(), newMw.Subscribe(testChannel)) - idList := flow.IdentityList(append(m.ids, newId)) - // needed to enable ID translation m.providers[0].SetIdentities(idList) // update the addresses m.mws[0].UpdateNodeAddresses() - // send 6 unicast messages, 5 should be allowed and the 6th should be rate limited - for i := 0; i < 6; i++ { + // add our sender node as a direct peer to our receiving node, this allows us to ensure + // that connections to peers that are rate limited are completely prune. IsConnected will + // return true only if the node is a direct peer of the other, after rate limiting this direct + // peer should be removed by the peer manager. + err = libP2PNodes[0].AddPeer(ctx, m.nodes[0].Host().Peerstore().PeerInfo(expectedPID)) + require.NoError(m.T(), err) + + p2ptest.EnsureConnected(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0], m.nodes[0]}) + + // with the rate limit configured to 5 msg/sec we send 10 messages at once and expect the rate limiter + // to be invoked at-least once. We send 10 messages due to the flakiness that is caused by async stream + // handling of streams. + for i := 0; i < 10; i++ { msg, err := network.NewOutgoingScope( flow.IdentifierList{newId.NodeID}, testChannel, @@ -308,15 +331,36 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { Text: fmt.Sprintf("hello-%d", i), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) - err = m.mws[0].SendDirect(msg) + err = m.mws[0].SendDirect(msg) require.NoError(m.T(), err) } // wait for all rate limits before shutting down middleware - unittest.RequireCloseBefore(m.T(), ch, 100*time.Millisecond, "could not stop on rate limit test ch on time") + unittest.RequireCloseBefore(m.T(), ch, 100*time.Millisecond, "could not stop rate limit test ch on time") + + // sleep for 1 seconds to allow connection pruner to prune connections + time.Sleep(1 * time.Second) + + // ensure connection to rate limited peer is pruned + p2pfixtures.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + p2pfixtures.EnsureNoStreamCreationBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + + // eventually the rate limited node should be able to reconnect and send messages + require.Eventually(m.T(), func() bool { + msg, err := network.NewOutgoingScope( + flow.IdentifierList{newId.NodeID}, + testChannel, + &libp2pmessage.TestMessage{ + Text: "hello", + }, + unittest.NetworkCodec().Encode, + message.ProtocolTypeUnicast) + require.NoError(m.T(), err) + return m.mws[0].SendDirect(msg) == nil + }, 3*time.Second, 100*time.Millisecond) // shutdown our middleware so that each message can be processed cancel() @@ -324,30 +368,29 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Messages() { unittest.RequireCloseBefore(m.T(), newMw.Done(), 100*time.Millisecond, "could not stop middleware on time") // expect our rate limited peer callback to be invoked once - require.Equal(m.T(), uint64(1), rateLimits.Load()) + require.True(m.T(), rateLimits.Load() > 0) } func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { - unittest.SkipUnless(m.T(), unittest.TEST_FLAKY, "disabling so that flaky metrics can be gathered before re-enabling") - //limiter limit will be set up to 1000 bytes/sec limit := rate.Limit(1000) //burst per interval burst := 1000 - // create test time - testtime := unittest.NewTestTime() + // we only expect messages from the first middleware on the test suite + expectedPID, err := unittest.PeerIDFromFlowID(m.ids[0]) + require.NoError(m.T(), err) // setup bandwidth rate limiter - bandwidthRateLimiter := ratelimit.NewBandWidthRateLimiter(limit, burst, 1, p2p.WithGetTimeNowFunc(testtime.Now)) + bandwidthRateLimiter := ratelimit.NewBandWidthRateLimiter(limit, burst, 3) - // the onUnicastRateLimitedPeerFunc call back we will use to keep track of how many times a rate limit happens + // the onRateLimit call back we will use to keep track of how many times a rate limit happens // after 5 rate limits we will close ch. ch := make(chan struct{}) rateLimits := atomic.NewUint64(0) - onRateLimit := func(peerID peer.ID, role, msgType string, topic channels.Topic, reason ratelimit.RateLimitReason) { - require.Equal(m.T(), reason, ratelimit.ReasonBandwidth) + onRateLimit := func(peerID peer.ID, role, msgType, topic, reason string) { + require.Equal(m.T(), reason, ratelimit.ReasonBandwidth.String()) // we only expect messages from the first middleware on the test suite expectedPID, err := unittest.PeerIDFromFlowID(m.ids[0]) @@ -358,28 +401,43 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { close(ch) } - rateLimiters := ratelimit.NewRateLimiters(&ratelimit.NoopRateLimiter{}, - bandwidthRateLimiter, - onRateLimit, - ratelimit.WithDisabledRateLimiting(false)) + consumer := testutils.NewRateLimiterConsumer(onRateLimit) + distributor := ratelimit.NewUnicastRateLimiterDistributor() + distributor.AddConsumer(consumer) + opts := []ratelimit.RateLimitersOption{ratelimit.WithBandwidthRateLimiter(bandwidthRateLimiter), ratelimit.WithNotifier(distributor), ratelimit.WithDisabledRateLimiting(false)} + rateLimiters := ratelimit.NewRateLimiters(opts...) + + idProvider := testutils.NewUpdatableIDProvider(m.ids) + // create connection gater, connection gater will refuse connections from rate limited nodes + connGater := testutils.NewConnectionGater(idProvider, func(pid peer.ID) error { + if bandwidthRateLimiter.IsRateLimited(pid) { + return fmt.Errorf("rate-limited peer") + } + return nil + }) // create a new staked identity - ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), m.logger, 1) + ids, libP2PNodes, _ := testutils.GenerateIDs(m.T(), + m.logger, + 1, + testutils.WithUnicastRateLimiterDistributor(distributor), + testutils.WithConnectionGater(connGater)) + idProvider.SetIdentities(append(m.ids, ids...)) // create middleware - opts := testutils.WithUnicastRateLimiters(rateLimiters) mws, providers := testutils.GenerateMiddlewares(m.T(), m.logger, ids, libP2PNodes, unittest.NetworkCodec(), - m.slashingViolationsConsumer, opts) + m.slashingViolationsConsumer, + testutils.WithUnicastRateLimiters(rateLimiters), + testutils.WithPeerManagerFilters(testutils.IsRateLimitedPeerFilter(bandwidthRateLimiter))) require.Len(m.T(), ids, 1) require.Len(m.T(), providers, 1) require.Len(m.T(), mws, 1) newId := ids[0] newMw := mws[0] - overlay := m.createOverlay(providers[0]) overlay.On("Receive", m.ids[0].NodeID, mockery.AnythingOfType("*message.Message")).Return(nil) @@ -414,29 +472,51 @@ func (m *MiddlewareTestSuite) TestUnicastRateLimit_Bandwidth() { Text: string(b), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // update the addresses m.mws[0].UpdateNodeAddresses() - // for the duration of a simulated second we will send 3 messages. Each message is about - // 400 bytes, the 3rd message will put our limiter over the 1000 byte limit at 1200 bytes. Thus - // the 3rd message should be rate limited. - start := testtime.Now() - end := start.Add(time.Second) - for testtime.Now().Before(end) { + // add our sender node as a direct peer to our receiving node, this allows us to ensure + // that connections to peers that are rate limited are completely prune. IsConnected will + // return true only if the node is a direct peer of the other, after rate limiting this direct + // peer should be removed by the peer manager. + err = libP2PNodes[0].AddPeer(ctx, m.nodes[0].Host().Peerstore().PeerInfo(expectedPID)) + require.NoError(m.T(), err) + p2ptest.EnsureConnected(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0], m.nodes[0]}) + // send 3 messages at once with a size of 400 bytes each. The third message will be rate limited + // as it is more than our allowed bandwidth of 1000 bytes. + for i := 0; i < 3; i++ { err := m.mws[0].SendDirect(msg) require.NoError(m.T(), err) - - // send 3 messages - testtime.Advance(334 * time.Millisecond) } // wait for all rate limits before shutting down middleware unittest.RequireCloseBefore(m.T(), ch, 100*time.Millisecond, "could not stop on rate limit test ch on time") + // sleep for 1 seconds to allow connection pruner to prune connections + time.Sleep(1 * time.Second) + + // ensure connection to rate limited peer is pruned + p2pfixtures.EnsureNotConnectedBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + p2pfixtures.EnsureNoStreamCreationBetweenGroups(m.T(), ctx, []p2p.LibP2PNode{libP2PNodes[0]}, []p2p.LibP2PNode{m.nodes[0]}) + + // eventually the rate limited node should be able to reconnect and send messages + require.Eventually(m.T(), func() bool { + msg, err := network.NewOutgoingScope( + flow.IdentifierList{newId.NodeID}, + testChannel, + &libp2pmessage.TestMessage{ + Text: "", + }, + unittest.NetworkCodec().Encode, + message.ProtocolTypeUnicast) + require.NoError(m.T(), err) + return m.mws[0].SendDirect(msg) == nil + }, 3*time.Second, 100*time.Millisecond) + // shutdown our middleware so that each message can be processed cancel() unittest.RequireComponentsDoneBefore(m.T(), 100*time.Millisecond, newMw) @@ -494,7 +574,7 @@ func (m *MiddlewareTestSuite) TestPing() { Text: expectedPayload, }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) m.ov[lastNodeIndex].On("Receive", mockery.Anything).Return(nil).Once(). @@ -507,7 +587,7 @@ func (m *MiddlewareTestSuite) TestPing() { require.Equal(m.T(), testChannel, msg.Channel()) // channel require.Equal(m.T(), m.ids[firstNodeIndex].NodeID, msg.OriginId()) // sender id require.Equal(m.T(), m.ids[lastNodeIndex].NodeID, msg.TargetIDs()[0]) // target id - require.Equal(m.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(m.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol require.Equal(m.T(), expectedPayload, msg.DecodedPayload().(*libp2pmessage.TestMessage).Text) // payload }) @@ -552,7 +632,7 @@ func (m *MiddlewareTestSuite) MultiPing(count int) { Text: expectedPayloadText, }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) m.ov[lastNodeIndex].On("Receive", mockery.Anything).Return(nil).Once(). @@ -565,7 +645,7 @@ func (m *MiddlewareTestSuite) MultiPing(count int) { require.Equal(m.T(), testChannel, msg.Channel()) // channel require.Equal(m.T(), m.ids[firstNodeIndex].NodeID, msg.OriginId()) // sender id require.Equal(m.T(), m.ids[lastNodeIndex].NodeID, msg.TargetIDs()[0]) // target id - require.Equal(m.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(m.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol // payload decodedPayload := msg.DecodedPayload().(*libp2pmessage.TestMessage).Text @@ -615,7 +695,7 @@ func (m *MiddlewareTestSuite) TestEcho() { Text: expectedSendMsg, }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // reply from last node to the first node. @@ -627,7 +707,7 @@ func (m *MiddlewareTestSuite) TestEcho() { Text: expectedReplyMsg, }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // last node @@ -642,7 +722,7 @@ func (m *MiddlewareTestSuite) TestEcho() { require.Equal(m.T(), testChannel, msg.Channel()) // channel require.Equal(m.T(), m.ids[first].NodeID, msg.OriginId()) // sender id require.Equal(m.T(), lastNode, msg.TargetIDs()[0]) // target id - require.Equal(m.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(m.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol require.Equal(m.T(), expectedSendMsg, msg.DecodedPayload().(*libp2pmessage.TestMessage).Text) // payload // event id eventId, err := network.EventId(msg.Channel(), msg.Proto().Payload) @@ -665,7 +745,7 @@ func (m *MiddlewareTestSuite) TestEcho() { require.Equal(m.T(), testChannel, msg.Channel()) // channel require.Equal(m.T(), m.ids[last].NodeID, msg.OriginId()) // sender id require.Equal(m.T(), firstNode, msg.TargetIDs()[0]) // target id - require.Equal(m.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(m.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol require.Equal(m.T(), expectedReplyMsg, msg.DecodedPayload().(*libp2pmessage.TestMessage).Text) // payload // event id eventId, err := network.EventId(msg.Channel(), msg.Proto().Payload) @@ -707,7 +787,7 @@ func (m *MiddlewareTestSuite) TestMaxMessageSize_SendDirect() { testChannel, event, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // sends a direct message from first node to the last node @@ -735,7 +815,7 @@ func (m *MiddlewareTestSuite) TestLargeMessageSize_SendDirect() { channels.ProvideChunks, event, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) // expect one message to be received by the target @@ -748,7 +828,7 @@ func (m *MiddlewareTestSuite) TestLargeMessageSize_SendDirect() { require.Equal(m.T(), channels.ProvideChunks, msg.Channel()) require.Equal(m.T(), m.ids[sourceIndex].NodeID, msg.OriginId()) require.Equal(m.T(), targetNode, msg.TargetIDs()[0]) - require.Equal(m.T(), network.ProtocolTypeUnicast, msg.Protocol()) + require.Equal(m.T(), message.ProtocolTypeUnicast, msg.Protocol()) eventId, err := network.EventId(msg.Channel(), msg.Proto().Payload) require.NoError(m.T(), err) @@ -788,7 +868,7 @@ func (m *MiddlewareTestSuite) TestMaxMessageSize_Publish() { testChannel, event, unittest.NetworkCodec().Encode, - network.ProtocolTypePubSub) + message.ProtocolTypePubSub) require.NoError(m.T(), err) // sends a direct message from first node to the last node @@ -825,7 +905,7 @@ func (m *MiddlewareTestSuite) TestUnsubscribe() { Text: string("hello1"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) m.ov[last].On("Receive", mockery.Anything).Return(nil).Run(func(args mockery.Arguments) { @@ -853,7 +933,7 @@ func (m *MiddlewareTestSuite) TestUnsubscribe() { Text: string("hello2"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(m.T(), err) err = m.mws[first].Publish(message2) diff --git a/network/test/unicast_authorization_test.go b/network/test/unicast_authorization_test.go index c536a53155a..6fe4d0b8b58 100644 --- a/network/test/unicast_authorization_test.go +++ b/network/test/unicast_authorization_test.go @@ -2,7 +2,6 @@ package test import ( "context" - "fmt" "testing" "time" @@ -129,7 +128,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type Channel: channels.TestNetworkChannel, // message will not be decoded before OnSenderEjectedError is logged, we won't log peer ID - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: validator.ErrIdentityUnverified, } slashingViolationsConsumer.On( @@ -164,7 +163,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() Text: string("hello"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -191,7 +190,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type Channel: channels.TestNetworkChannel, // message will not be decoded before OnSenderEjectedError is logged, we won't log peer ID - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: validator.ErrSenderEjected, } slashingViolationsConsumer.On( @@ -225,7 +224,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { Text: string("hello"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -248,9 +247,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee expectedViolation := &slashing.Violation{ Identity: u.senderID, PeerID: expectedSenderPeerID.String(), - MsgType: message.TestMessage, + MsgType: "*message.TestMessage", Channel: channels.ConsensusCommittee, - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: message.ErrUnauthorizedMessageOnChannel, } @@ -285,7 +284,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee Text: string("hello"), }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -305,7 +304,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - invalidMessageCode := byte('X') + invalidMessageCode := codec.MessageCode(byte('X')) var nilID *flow.Identity expectedViolation := &slashing.Violation{ @@ -313,7 +312,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( PeerID: expectedSenderPeerID.String(), MsgType: "", Channel: channels.TestNetworkChannel, - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: codec.NewUnknownMsgCodeErr(invalidMessageCode), } @@ -352,10 +351,10 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( e, err := unittest.NetworkCodec().Encode(msg) require.NoError(u.T(), err) // manipulate message code byte - e[0] = invalidMessageCode + e[0] = invalidMessageCode.Uint8() return e, nil }, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -377,19 +376,17 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() modifiedMessageCode := codec.CodeDKGMessage - var nilID *flow.Identity expectedViolation := &slashing.Violation{ - Identity: nilID, + Identity: u.senderID, PeerID: expectedSenderPeerID.String(), - MsgType: "", + MsgType: "*messages.DKGMessage", Channel: channels.TestNetworkChannel, - Protocol: message.ProtocolUnicast, - //NOTE: in this test the message code does not match the underlying message type causing the codec to fail to unmarshal the message when decoding. - Err: codec.NewMsgUnmarshalErr(modifiedMessageCode, message.DKGMessage, fmt.Errorf("cbor: found unknown field at map element index 0")), + Protocol: message.ProtocolTypeUnicast, + Err: message.ErrUnauthorizedMessageOnChannel, } slashingViolationsConsumer.On( - "OnInvalidMsgError", + "OnUnAuthorizedSenderError", expectedViolation, ).Once().Run(func(args mockery.Arguments) { close(u.waitCh) @@ -423,10 +420,10 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() e, err := unittest.NetworkCodec().Encode(msg) require.NoError(u.T(), err) // manipulate message code byte - e[0] = modifiedMessageCode + e[0] = modifiedMessageCode.Uint8() return e, nil }, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -451,7 +448,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_PublicChannel() Text: expectedPayload, }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) overlay := mocknetwork.NewOverlay(u.T()) @@ -475,7 +472,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_PublicChannel() require.Equal(u.T(), testChannel, msg.Channel()) // channel require.Equal(u.T(), u.senderID.NodeID, msg.OriginId()) // sender id require.Equal(u.T(), u.receiverID.NodeID, msg.TargetIDs()[0]) // target id - require.Equal(u.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(u.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol require.Equal(u.T(), expectedPayload, msg.DecodedPayload().(*libp2pmessage.TestMessage).Text) // payload }) @@ -507,9 +504,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni expectedViolation := &slashing.Violation{ Identity: u.senderID, PeerID: expectedSenderPeerID.String(), - MsgType: "BlockProposal", + MsgType: "*messages.BlockProposal", Channel: channels.ConsensusCommittee, - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: message.ErrUnauthorizedUnicastOnChannel, } @@ -546,7 +543,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni channel, payload, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -570,9 +567,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu expectedViolation := &slashing.Violation{ Identity: nil, PeerID: expectedSenderPeerID.String(), - MsgType: message.TestMessage, + MsgType: "*message.TestMessage", Channel: channels.TestNetworkChannel, - Protocol: message.ProtocolUnicast, + Protocol: message.ProtocolTypeUnicast, Err: middleware.ErrUnicastMsgWithoutSub, } @@ -605,7 +602,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu Text: "TestUnicastAuthorization_ReceiverHasNoSubscription", }, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) // send message via unicast @@ -629,7 +626,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasSubs channel, &messages.EntityRequest{}, unittest.NetworkCodec().Encode, - network.ProtocolTypeUnicast) + message.ProtocolTypeUnicast) require.NoError(u.T(), err) u.senderID.Role = flow.RoleConsensus @@ -656,7 +653,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasSubs require.Equal(u.T(), channel, msg.Channel()) // channel require.Equal(u.T(), u.senderID.NodeID, msg.OriginId()) // sender id require.Equal(u.T(), u.receiverID.NodeID, msg.TargetIDs()[0]) // target id - require.Equal(u.T(), network.ProtocolTypeUnicast, msg.Protocol()) // protocol + require.Equal(u.T(), message.ProtocolTypeUnicast, msg.Protocol()) // protocol }) u.startMiddlewares(overlay) diff --git a/network/validator/authorized_sender_validator.go b/network/validator/authorized_sender_validator.go index ba6371b1744..0af21b45e39 100644 --- a/network/validator/authorized_sender_validator.go +++ b/network/validator/authorized_sender_validator.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/slashing" @@ -39,8 +40,8 @@ func NewAuthorizedSenderValidator(log zerolog.Logger, slashingViolationsConsumer // PubSubMessageValidator wraps Validate and returns PubSubMessageValidator callback that returns pubsub.ValidationReject if validation fails and pubsub.ValidationAccept if validation passes. func (av *AuthorizedSenderValidator) PubSubMessageValidator(channel channels.Channel) PubSubMessageValidator { - return func(from peer.ID, msg interface{}) p2p.ValidationResult { - _, err := av.Validate(from, msg, channel, message.ProtocolPublish) + return func(from peer.ID, msg *message.Message) p2p.ValidationResult { + _, err := av.Validate(from, msg.Payload, channel, message.ProtocolTypePubSub) if err != nil { return p2p.ValidationReject } @@ -54,7 +55,7 @@ func (av *AuthorizedSenderValidator) PubSubMessageValidator(channel channels.Cha // Otherwise, the message is rejected. The message is also authorized by checking that the sender is allowed to send the message on the channel. // If validation fails the message is rejected, and if the validation error is an expected error, slashing data is also collected. // Authorization config is defined in message.MsgAuthConfig. -func (av *AuthorizedSenderValidator) Validate(from peer.ID, msg interface{}, channel channels.Channel, protocol message.Protocol) (string, error) { +func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, channel channels.Channel, protocol message.ProtocolType) (string, error) { // NOTE: Gossipsub messages from unstaked nodes should be rejected by the libP2P node topic validator // before they reach message validators. If a message from a unstaked peer gets to this point // something terrible went wrong. @@ -65,8 +66,14 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, msg interface{}, cha return "", ErrIdentityUnverified } - msgType, err := av.isAuthorizedSender(identity, channel, msg, protocol) + msgCode, err := codec.MessageCodeFromPayload(payload) + if err != nil { + violation := &slashing.Violation{Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} + av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + return "", err + } + msgType, err := av.isAuthorizedSender(identity, channel, msgCode, protocol) switch { case err == nil: return msgType, nil @@ -105,18 +112,26 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, msg interface{}, cha // // Expected error returns during normal operations: // - ErrSenderEjected: if identity of sender is ejected from the network -// - message.ErrUnknownMsgType if message auth config us not found for the msg +// - codec.ErrUnknownMsgCode: if interface for the encoded message code byte is unknown +// - message.UnknownMsgTypeErr if message auth config us not found for the msg // - message.ErrUnauthorizedMessageOnChannel if msg is not authorized to be sent on channel // - message.ErrUnauthorizedRole if sender role is not authorized to send msg -func (av *AuthorizedSenderValidator) isAuthorizedSender(identity *flow.Identity, channel channels.Channel, msg interface{}, protocol message.Protocol) (string, error) { +func (av *AuthorizedSenderValidator) isAuthorizedSender(identity *flow.Identity, channel channels.Channel, msgCode codec.MessageCode, protocol message.ProtocolType) (string, error) { if identity.Ejected { return "", ErrSenderEjected } + // attempt to get the message interface from the message code encoded into the first byte of the message payload + // this will be used to get the message auth configuration. + msgInterface, what, err := codec.InterfaceFromMessageCode(msgCode) + if err != nil { + return "", fmt.Errorf("could not extract interface from message code %v: %w", msgCode, err) + } + // get message auth config - conf, err := message.GetMessageAuthConfig(msg) + conf, err := message.GetMessageAuthConfig(msgInterface) if err != nil { - return "", err + return "", fmt.Errorf("could not get authorization config for interface %T: %w", msgInterface, err) } // handle special case for cluster prefixed channels @@ -125,8 +140,8 @@ func (av *AuthorizedSenderValidator) isAuthorizedSender(identity *flow.Identity, } if err := conf.EnsureAuthorized(identity.Role, channel, protocol); err != nil { - return conf.Name, err + return what, err } - return conf.Name, nil + return what, nil } diff --git a/network/validator/authorized_sender_validator_test.go b/network/validator/authorized_sender_validator_test.go index 7462a679899..966ae5ba127 100644 --- a/network/validator/authorized_sender_validator_test.go +++ b/network/validator/authorized_sender_validator_test.go @@ -12,7 +12,9 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/slashing" @@ -24,6 +26,7 @@ type TestCase struct { GetIdentity func(pid peer.ID) (*flow.Identity, bool) Channel channels.Channel Message interface{} + MessageCode codec.MessageCode MessageStr string Protocols message.Protocols } @@ -42,6 +45,7 @@ type TestAuthorizedSenderValidatorSuite struct { log zerolog.Logger slashingViolationsConsumer slashing.ViolationsConsumer allMsgConfigs []message.MsgAuthConfig + codec network.Codec } func (s *TestAuthorizedSenderValidatorSuite) SetupTest() { @@ -51,6 +55,7 @@ func (s *TestAuthorizedSenderValidatorSuite) SetupTest() { s.initializeUnicastOnChannelTestCases() s.log = unittest.Logger() s.slashingViolationsConsumer = slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector()) + s.codec = unittest.NetworkCodec() } // TestValidatorCallback_AuthorizedSender checks that AuthorizedSenderValidator.Validate does not return false positive @@ -66,8 +71,8 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen // ensure according to the message auth config, if a message is authorized to be sent via unicast it // is accepted or rejected. - msgType, err := authorizedSenderValidator.Validate(pid, c.Message, c.Channel, message.ProtocolUnicast) - if c.Protocols.Contains(message.ProtocolUnicast) { + msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) + if c.Protocols.Contains(message.ProtocolTypeUnicast) { require.NoError(s.T(), err) require.Equal(s.T(), c.MessageStr, msgType) } else { @@ -75,9 +80,15 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen require.Equal(s.T(), c.MessageStr, msgType) } + payload, err := s.codec.Encode(c.Message) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: c.Channel.String(), + Payload: payload, + } validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) - pubsubResult := validatePubsub(pid, c.Message) - if !c.Protocols.Contains(message.ProtocolPublish) { + pubsubResult := validatePubsub(pid, m) + if !c.Protocols.Contains(message.ProtocolTypePubSub) { require.Equal(s.T(), p2p.ValidationReject, pubsubResult) } else { require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) @@ -97,8 +108,14 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedS authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + payload, err := s.codec.Encode(c.Message) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: c.Channel.String(), + Payload: payload, + } validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) - pubsubResult := validatePubsub(pid, c.Message) + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationReject, pubsubResult) }) } @@ -115,7 +132,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedUni authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - msgType, err := authorizedSenderValidator.Validate(pid, c.Message, c.Channel, message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.NoError(s.T(), err) require.Equal(s.T(), c.MessageStr, msgType) }) @@ -133,7 +150,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedU authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - msgType, err := authorizedSenderValidator.Validate(pid, c.Message, c.Channel, message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) require.Equal(s.T(), c.MessageStr, msgType) }) @@ -151,12 +168,18 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedM authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - msgType, err := authorizedSenderValidator.Validate(pid, c.Message, c.Channel, message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedMessageOnChannel) require.Equal(s.T(), c.MessageStr, msgType) + payload, err := s.codec.Encode(c.Message) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: c.Channel.String(), + Payload: payload, + } validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) - pubsubResult := validatePubsub(pid, c.Message) + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationReject, pubsubResult) }) } @@ -175,23 +198,35 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) // ensure ClusterBlockProposal not allowed to be sent on channel via unicast - msgType, err := authorizedSenderValidator.Validate(pid, &messages.ClusterBlockProposal{}, channels.ConsensusCluster(clusterID), message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeClusterBlockProposal.Uint8()}, channels.ConsensusCluster(clusterID), message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), message.ClusterBlockProposal, msgType) + require.Equal(s.T(), "*messages.ClusterBlockProposal", msgType) // ensure ClusterBlockProposal is allowed to be sent via pubsub by authorized sender + payload, err := s.codec.Encode(&messages.ClusterBlockProposal{}) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: channels.ConsensusCluster(clusterID).String(), + Payload: payload, + } validateCollConsensusPubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCluster(clusterID)) - pubsubResult := validateCollConsensusPubsub(pid, &messages.ClusterBlockProposal{}) + pubsubResult := validateCollConsensusPubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast - msgType, err = authorizedSenderValidator.Validate(pid, &messages.SyncRequest{}, channels.SyncCluster(clusterID), message.ProtocolUnicast) + msgType, err = authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCluster(clusterID), message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), message.SyncRequest, msgType) + require.Equal(s.T(), "*messages.SyncRequest", msgType) // ensure SyncRequest is allowed to be sent via pubsub by authorized sender + payload, err = s.codec.Encode(&messages.SyncRequest{}) + require.NoError(s.T(), err) + m = &message.Message{ + ChannelID: channels.SyncCluster(clusterID).String(), + Payload: payload, + } validateSyncClusterPubsub := authorizedSenderValidator.PubSubMessageValidator(channels.SyncCluster(clusterID)) - pubsubResult = validateSyncClusterPubsub(pid, &messages.SyncRequest{}) + pubsubResult = validateSyncClusterPubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) } @@ -206,23 +241,23 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) - msgType, err := authorizedSenderValidator.Validate(pid, &messages.SyncRequest{}, channels.SyncCommittee, message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrSenderEjected) require.Equal(s.T(), "", msgType) + payload, err := s.codec.Encode(&messages.SyncRequest{}) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: channels.SyncCommittee.String(), + Payload: payload, + } validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.SyncCommittee) - pubsubResult := validatePubsub(pid, &messages.SyncRequest{}) + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationReject, pubsubResult) }) - s.Run("unknown message type", func() { + s.Run("unknown message code", func() { identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(flow.RoleConsensus)) - type msg struct { - *messages.BlockProposal - } - - // *validator.msg is not a known message type, but embeds *messages.BlockProposal which is - m := &msg{unittest.ProposalFixture()} getIdentityFunc := s.getIdentity(identity) pid, err := unittest.PeerIDFromFlowID(identity) @@ -232,17 +267,18 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCommittee) // unknown message types are rejected - msgType, err := authorizedSenderValidator.Validate(pid, m, channels.ConsensusCommittee, message.ProtocolUnicast) - require.True(s.T(), message.IsUnknownMsgTypeErr(err)) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{'x'}, channels.ConsensusCommittee, message.ProtocolTypeUnicast) + require.True(s.T(), codec.IsErrUnknownMsgCode(err)) require.Equal(s.T(), "", msgType) - pubsubResult := validatePubsub(pid, m) - require.Equal(s.T(), p2p.ValidationReject, pubsubResult) - // nil messages are rejected - msgType, err = authorizedSenderValidator.Validate(pid, nil, channels.ConsensusCommittee, message.ProtocolUnicast) - require.True(s.T(), message.IsUnknownMsgTypeErr(err)) - require.Equal(s.T(), "", msgType) - pubsubResult = validatePubsub(pid, nil) + payload, err := s.codec.Encode(&messages.BlockProposal{}) + require.NoError(s.T(), err) + payload[0] = byte('x') + netMsg := &message.Message{ + ChannelID: channels.ConsensusCommittee.String(), + Payload: payload, + } + pubsubResult := validatePubsub(pid, netMsg) require.Equal(s.T(), p2p.ValidationReject, pubsubResult) }) @@ -257,12 +293,18 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) - msgType, err := authorizedSenderValidator.Validate(pid, &messages.SyncRequest{}, channels.SyncCommittee, message.ProtocolUnicast) + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrIdentityUnverified) require.Equal(s.T(), "", msgType) + payload, err := s.codec.Encode(&messages.SyncRequest{}) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: channels.SyncCommittee.String(), + Payload: payload, + } validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.SyncCommittee) - pubsubResult := validatePubsub(pid, &messages.SyncRequest{}) + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationReject, pubsubResult) }) } @@ -276,9 +318,8 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnauthorizedP require.NoError(s.T(), err) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - - msgType, err := authorizedSenderValidator.Validate(pid, c.Message, c.Channel, message.ProtocolPublish) - if c.MessageStr == message.TestMessage { + msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypePubSub) + if c.MessageStr == "*message.TestMessage" { require.NoError(s.T(), err) } else { require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) @@ -294,12 +335,15 @@ func (s *TestAuthorizedSenderValidatorSuite) initializeAuthorizationTestCases() for channel, channelAuthConfig := range c.Config { for _, role := range flow.Roles() { identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(role)) + code, what, err := codec.MessageCodeFromInterface(c.Type()) + require.NoError(s.T(), err) tc := TestCase{ Identity: identity, GetIdentity: s.getIdentity(identity), Channel: channel, Message: c.Type(), - MessageStr: c.Name, + MessageCode: code, + MessageStr: what, Protocols: channelAuthConfig.AllowedProtocols, } if channelAuthConfig.AuthorizedRoles.Contains(role) { @@ -326,13 +370,16 @@ func (s *TestAuthorizedSenderValidatorSuite) initializeInvalidMessageOnChannelTe for _, config := range s.allMsgConfigs { // include test if message type is not authorized on channel _, ok := config.Config[channel] + code, what, err := codec.MessageCodeFromInterface(config.Type()) + require.NoError(s.T(), err) if config.Name != c.Name && !ok { tc := TestCase{ Identity: identity, GetIdentity: s.getIdentity(identity), Channel: channel, Message: config.Type(), - MessageStr: config.Name, + MessageCode: code, + MessageStr: what, Protocols: channelAuthConfig.AllowedProtocols, } s.unauthorizedMessageOnChannelTestCases = append(s.unauthorizedMessageOnChannelTestCases, tc) @@ -347,15 +394,18 @@ func (s *TestAuthorizedSenderValidatorSuite) initializeUnicastOnChannelTestCases for _, c := range s.allMsgConfigs { for channel, channelAuthConfig := range c.Config { identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(channelAuthConfig.AuthorizedRoles[0])) + code, what, err := codec.MessageCodeFromInterface(c.Type()) + require.NoError(s.T(), err) tc := TestCase{ Identity: identity, GetIdentity: s.getIdentity(identity), Channel: channel, Message: c.Type(), - MessageStr: c.Name, + MessageCode: code, + MessageStr: what, Protocols: channelAuthConfig.AllowedProtocols, } - if channelAuthConfig.AllowedProtocols.Contains(message.ProtocolUnicast) { + if channelAuthConfig.AllowedProtocols.Contains(message.ProtocolTypeUnicast) { s.authorizedUnicastOnChannel = append(s.authorizedUnicastOnChannel, tc) } else { s.unauthorizedUnicastOnChannel = append(s.unauthorizedUnicastOnChannel, tc) diff --git a/network/validator/pubsub/topic_validator.go b/network/validator/pubsub/topic_validator.go index c97eebdf08c..954c5f6b401 100644 --- a/network/validator/pubsub/topic_validator.go +++ b/network/validator/pubsub/topic_validator.go @@ -10,12 +10,8 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/network/channels" - "github.com/onflow/flow-go/network/codec" - "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/slashing" - - "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/validator" _ "github.com/onflow/flow-go/utils/binstat" "github.com/onflow/flow-go/utils/logging" @@ -62,14 +58,13 @@ func messageSigningID(m *pubsub.Message) (peer.ID, error) { // TopicValidatorData includes information about the message being sent. type TopicValidatorData struct { - Message *message.Message - DecodedMsgPayload interface{} - From peer.ID + Message *message.Message + From peer.ID } // TopicValidator is the topic validator that is registered with libP2P whenever a flow libP2P node subscribes to a topic. -// The TopicValidator will decode and perform validation on the raw pubsub message. -func TopicValidator(log zerolog.Logger, c network.Codec, slashingViolationsConsumer slashing.ViolationsConsumer, peerFilter func(peer.ID) error, validators ...validator.PubSubMessageValidator) p2p.TopicValidatorFunc { +// The TopicValidator will perform validation on the raw pubsub message. +func TopicValidator(log zerolog.Logger, peerFilter func(peer.ID) error, validators ...validator.PubSubMessageValidator) p2p.TopicValidatorFunc { log = log.With(). Str("component", "libp2p-node-topic-validator"). Logger() @@ -126,38 +121,14 @@ func TopicValidator(log zerolog.Logger, c network.Codec, slashingViolationsConsu return p2p.ValidationReject } - // Convert message payload to a known message type - decodedMsgPayload, err := c.Decode(msg.Payload) - switch { - case err == nil: - break - case codec.IsErrUnknownMsgCode(err): - // slash peer if message contains unknown message code byte - slashingViolationsConsumer.OnUnknownMsgTypeError(violation(from, channel, err)) - return p2p.ValidationReject - case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): - // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte - slashingViolationsConsumer.OnInvalidMsgError(violation(from, channel, err)) - return p2p.ValidationReject - default: - // unexpected error condition. this indicates there's a bug - // don't crash as a result of external inputs since that creates a DoS vector. - lg.Error(). - Err(fmt.Errorf("unexpected error while decoding message: %w", err)). - Bool(logging.KeySuspicious, true). - Msg("rejecting message") - return p2p.ValidationReject - } - rawMsg.ValidatorData = TopicValidatorData{ - Message: &msg, - DecodedMsgPayload: decodedMsgPayload, - From: from, + Message: &msg, + From: from, } result := p2p.ValidationAccept for _, v := range validators { - switch res := v(from, decodedMsgPayload); res { + switch res := v(from, &msg); res { case p2p.ValidationReject: return res case p2p.ValidationIgnore: @@ -168,12 +139,3 @@ func TopicValidator(log zerolog.Logger, c network.Codec, slashingViolationsConsu return result } } - -func violation(pid peer.ID, channel channels.Channel, err error) *slashing.Violation { - return &slashing.Violation{ - PeerID: pid.String(), - Channel: channel, - Protocol: message.ProtocolPublish, - Err: err, - } -} diff --git a/network/validator/validator.go b/network/validator/validator.go index c85cacf95cb..4ce1eba5f88 100644 --- a/network/validator/validator.go +++ b/network/validator/validator.go @@ -3,14 +3,10 @@ package validator import ( "github.com/libp2p/go-libp2p/core/peer" + "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" ) -// MessageValidator validates the given message with original sender `from` and returns an error if validation fails -// else upon successful validation it should return the decoded message type string. -// Note: contrarily to pubsub.ValidatorEx, the peerID parameter does not represent the bearer of the message, but its source. -type MessageValidator func(from peer.ID, msg interface{}) (string, error) - // PubSubMessageValidator validates the given message with original sender `from` and returns p2p.ValidationResult. // Note: contrarily to pubsub.ValidatorEx, the peerID parameter does not represent the bearer of the message, but its source. -type PubSubMessageValidator func(from peer.ID, msg interface{}) p2p.ValidationResult +type PubSubMessageValidator func(from peer.ID, msg *message.Message) p2p.ValidationResult diff --git a/scripts/update-core-contracts.sh b/scripts/update-core-contracts.sh new file mode 100755 index 00000000000..eb11bf433ab --- /dev/null +++ b/scripts/update-core-contracts.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# This script updates all flow-core-contracts dependencies to a new version. +# Specify the desired version as the only argument when running the script: +# ./scripts/update-core-contracts.sh v1.2.3 + +go get github.com/onflow/flow-core-contracts/lib/go/contracts@$1 +go get github.com/onflow/flow-core-contracts/lib/go/templates@$1 +cd integration +go get github.com/onflow/flow-core-contracts/lib/go/contracts@$1 +go get github.com/onflow/flow-core-contracts/lib/go/templates@$1 +cd .. diff --git a/state/cluster/badger/mutator.go b/state/cluster/badger/mutator.go index 7b29bcf4c26..a5c39142f00 100644 --- a/state/cluster/badger/mutator.go +++ b/state/cluster/badger/mutator.go @@ -36,7 +36,12 @@ func NewMutableState(state *State, tracer module.Tracer, headers storage.Headers return mutableState, nil } +// Extend validates that the given cluster block passes compliance rules, then inserts +// it to the cluster state. // TODO (Ramtin) pass context here +// Expected errors during normal operations: +// - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) +// - state.InvalidExtensionError if the candidate block is invalid func (m *MutableState) Extend(block *cluster.Block) error { blockID := block.ID() @@ -83,6 +88,12 @@ func (m *MutableState) Extend(block *cluster.Block) error { return fmt.Errorf("could not retrieve latest finalized header: %w", err) } + // extending block must have correct parent view + if header.ParentView != parent.View { + return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", + header.ParentView, parent.View) + } + // the extending block must increase height by 1 from parent if header.Height != parent.Height+1 { return state.NewInvalidExtensionErrorf("extending block height (%d) must be parent height + 1 (%d)", @@ -124,10 +135,10 @@ func (m *MutableState) Extend(block *cluster.Block) error { // NOTE: it is valid for a collection to be expired at this point, // otherwise we would compromise liveness of the cluster. refBlock, err := m.headers.ByBlockID(payload.ReferenceBlockID) - if errors.Is(err, storage.ErrNotFound) { - return state.NewUnverifiableExtensionError("cluster block references unknown reference block (id=%x)", payload.ReferenceBlockID) - } if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return state.NewUnverifiableExtensionError("cluster block references unknown reference block (id=%x)", payload.ReferenceBlockID) + } return fmt.Errorf("could not check reference block: %w", err) } diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 83f52d2a288..a81f4ebb248 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -65,7 +65,7 @@ func (suite *MutatorSuite) SetupTest() { headers, _, seals, index, conPayloads, blocks, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) - clusterStateRoot, err := NewStateRoot(suite.genesis) + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.NoError(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) suite.Assert().Nil(err) @@ -76,7 +76,7 @@ func (suite *MutatorSuite) SetupTest() { // just bootstrap with a genesis block, we'll use this as reference participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) genesis, result, seal := unittest.BootstrapFixture(participants) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(genesis.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(genesis.ID())) // ensure we don't enter a new epoch for tests that build many blocks result.ServiceEvents[0].Event.(*flow.EpochSetup).FinalView = genesis.Header.View + 100000 seal.ResultID = result.ID() @@ -168,21 +168,21 @@ func TestMutator(t *testing.T) { func (suite *MutatorSuite) TestBootstrap_InvalidNumber() { suite.genesis.Header.Height = 1 - _, err := NewStateRoot(suite.genesis) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Error(err) } func (suite *MutatorSuite) TestBootstrap_InvalidParentHash() { suite.genesis.Header.ParentID = unittest.IdentifierFixture() - _, err := NewStateRoot(suite.genesis) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Error(err) } func (suite *MutatorSuite) TestBootstrap_InvalidPayloadHash() { suite.genesis.Header.PayloadHash = unittest.IdentifierFixture() - _, err := NewStateRoot(suite.genesis) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Error(err) } @@ -190,7 +190,7 @@ func (suite *MutatorSuite) TestBootstrap_InvalidPayload() { // this is invalid because genesis collection should be empty suite.genesis.Payload = unittest.ClusterPayloadFixture(2) - _, err := NewStateRoot(suite.genesis) + _, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Error(err) } @@ -258,6 +258,18 @@ func (suite *MutatorSuite) TestExtend_InvalidBlockNumber() { suite.Assert().True(state.IsInvalidExtensionError(err)) } +// TestExtend_InvalidParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent +// with view of block referred by ParentID. +func (suite *MutatorSuite) TestExtend_InvalidParentView() { + block := suite.Block() + // change the block parent view + block.Header.ParentView-- + + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsInvalidExtensionError(err)) +} + func (suite *MutatorSuite) TestExtend_DuplicateTxInPayload() { block := suite.Block() // add the same transaction to a payload twice @@ -319,15 +331,25 @@ func (suite *MutatorSuite) TestExtend_WithEmptyCollection() { // an unknown reference block is unverifiable func (suite *MutatorSuite) TestExtend_WithNonExistentReferenceBlock() { - block := suite.Block() - tx := suite.Tx() - payload := suite.Payload(&tx) - // set a random reference block ID - payload.ReferenceBlockID = unittest.IdentifierFixture() - block.SetPayload(payload) - err := suite.state.Extend(&block) - suite.Assert().Error(err) - suite.Assert().True(state.IsUnverifiableExtensionError(err)) + suite.Run("empty collection", func() { + block := suite.Block() + block.Payload.ReferenceBlockID = unittest.IdentifierFixture() + block.SetPayload(*block.Payload) + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) + }) + suite.Run("non-empty collection", func() { + block := suite.Block() + tx := suite.Tx() + payload := suite.Payload(&tx) + // set a random reference block ID + payload.ReferenceBlockID = unittest.IdentifierFixture() + block.SetPayload(payload) + err := suite.state.Extend(&block) + suite.Assert().Error(err) + suite.Assert().True(state.IsUnverifiableExtensionError(err)) + }) } // a collection with an expired reference block is a VALID extension of chain state diff --git a/state/cluster/badger/snapshot_test.go b/state/cluster/badger/snapshot_test.go index 57a413115ac..d2dc9aee15b 100644 --- a/state/cluster/badger/snapshot_test.go +++ b/state/cluster/badger/snapshot_test.go @@ -58,7 +58,7 @@ func (suite *SnapshotSuite) SetupTest() { headers, _, seals, _, _, blocks, setups, commits, statuses, results := util.StorageLayer(suite.T(), suite.db) colPayloads := storage.NewClusterPayloads(metrics, suite.db) - clusterStateRoot, err := NewStateRoot(suite.genesis) + clusterStateRoot, err := NewStateRoot(suite.genesis, unittest.QuorumCertificateFixture()) suite.Assert().Nil(err) clusterState, err := Bootstrap(suite.db, clusterStateRoot) suite.Assert().Nil(err) diff --git a/state/cluster/badger/state.go b/state/cluster/badger/state.go index 8163063dc1b..33186a14b14 100644 --- a/state/cluster/badger/state.go +++ b/state/cluster/badger/state.go @@ -6,6 +6,7 @@ import ( "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/state/cluster" @@ -33,6 +34,7 @@ func Bootstrap(db *badger.DB, stateRoot *StateRoot) (*State, error) { state := newState(db, stateRoot.ClusterID()) genesis := stateRoot.Block() + rootQC := stateRoot.QC() // bootstrap cluster state err = operation.RetryOnConflict(state.db.Update, func(tx *badger.Txn) error { chainID := genesis.Header.ChainID @@ -52,14 +54,25 @@ func Bootstrap(db *badger.DB, stateRoot *StateRoot) (*State, error) { if err != nil { return fmt.Errorf("could not insert genesis boundary: %w", err) } - err = operation.InsertStartedView(chainID, genesis.Header.View)(tx) + + safetyData := &hotstuff.SafetyData{ + LockedOneChainView: genesis.Header.View, + HighestAcknowledgedView: genesis.Header.View, + } + + livenessData := &hotstuff.LivenessData{ + CurrentView: genesis.Header.View + 1, + NewestQC: rootQC, + } + // insert safety data + err = operation.InsertSafetyData(chainID, safetyData)(tx) if err != nil { - return fmt.Errorf("could not insert started view: %w", err) + return fmt.Errorf("could not insert safety data: %w", err) } - // insert voted view for hotstuff - err = operation.InsertVotedView(chainID, genesis.Header.View)(tx) + // insert liveness data + err = operation.InsertLivenessData(chainID, livenessData)(tx) if err != nil { - return fmt.Errorf("could not insert voted view: %w", err) + return fmt.Errorf("could not insert liveness data: %w", err) } return nil diff --git a/state/cluster/badger/state_root.go b/state/cluster/badger/state_root.go index 39e0203290d..e592ebd4a3c 100644 --- a/state/cluster/badger/state_root.go +++ b/state/cluster/badger/state_root.go @@ -10,15 +10,17 @@ import ( // StateRoot is the root information required to bootstrap the cluster state type StateRoot struct { block *cluster.Block + qc *flow.QuorumCertificate } -func NewStateRoot(genesis *cluster.Block) (*StateRoot, error) { +func NewStateRoot(genesis *cluster.Block, qc *flow.QuorumCertificate) (*StateRoot, error) { err := validateClusterGenesis(genesis) if err != nil { return nil, fmt.Errorf("inconsistent state root: %w", err) } return &StateRoot{ block: genesis, + qc: qc, }, nil } @@ -53,3 +55,7 @@ func (s StateRoot) ClusterID() flow.ChainID { func (s StateRoot) Block() *cluster.Block { return s.block } + +func (s StateRoot) QC() *flow.QuorumCertificate { + return s.qc +} diff --git a/state/cluster/state.go b/state/cluster/state.go index 06d79e608b1..19b58a64425 100644 --- a/state/cluster/state.go +++ b/state/cluster/state.go @@ -34,5 +34,8 @@ type MutableState interface { State // Extend introduces the given block into the cluster state as a pending // without modifying the current finalized state. + // Expected errors during normal operations: + // - state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) + // - state.InvalidExtensionError if the candidate block is invalid Extend(candidate *cluster.Block) error } diff --git a/state/errors.go b/state/errors.go index 92c144e613d..a62ca209200 100644 --- a/state/errors.go +++ b/state/errors.go @@ -90,28 +90,29 @@ func IsUnverifiableExtensionError(err error) bool { return errors.As(err, &errUnverifiableExtensionError) } -// NoValidChildBlockError is a sentinel error when the case where a certain block has -// no valid child. -type NoValidChildBlockError struct { +// NoChildBlockError is returned where a certain block has no valid child. +// Since all blocks are validated before being inserted to the state, this is +// equivalent to have no stored child. +type NoChildBlockError struct { error } -func NewNoValidChildBlockError(msg string) error { - return NoValidChildBlockError{ +func NewNoChildBlockError(msg string) error { + return NoChildBlockError{ error: fmt.Errorf(msg), } } -func NewNoValidChildBlockErrorf(msg string, args ...interface{}) error { - return NewNoValidChildBlockError(fmt.Sprintf(msg, args...)) +func NewNoChildBlockErrorf(msg string, args ...interface{}) error { + return NewNoChildBlockError(fmt.Sprintf(msg, args...)) } -func (e NoValidChildBlockError) Unwrap() error { +func (e NoChildBlockError) Unwrap() error { return e.error } -func IsNoValidChildBlockError(err error) bool { - return errors.As(err, &NoValidChildBlockError{}) +func IsNoChildBlockError(err error) bool { + return errors.As(err, &NoChildBlockError{}) } // UnknownBlockError is a sentinel error indicating that a certain block diff --git a/state/protocol/badger/config.go b/state/protocol/badger/config.go deleted file mode 100644 index e0708c699ce..00000000000 --- a/state/protocol/badger/config.go +++ /dev/null @@ -1,15 +0,0 @@ -package badger - -import ( - "github.com/onflow/flow-go/model/flow" -) - -type Config struct { - transactionExpiry uint64 // how many blocks after the reference block a transaction expires -} - -func DefaultConfig() Config { - return Config{ - transactionExpiry: flow.DefaultTransactionExpiry, - } -} diff --git a/state/protocol/badger/mutator.go b/state/protocol/badger/mutator.go index d05f2f2b0cb..b766bcf54c0 100644 --- a/state/protocol/badger/mutator.go +++ b/state/protocol/badger/mutator.go @@ -22,10 +22,6 @@ import ( "github.com/onflow/flow-go/storage/badger/transaction" ) -// errIncompleteEpochConfiguration is a sentinel error returned when there are -// still epoch service events missing and the new epoch can't be constructed. -var errIncompleteEpochConfiguration = errors.New("block beyond epoch boundary") - // FollowerState implements a lighter version of a mutable protocol state. // When extending the state, it performs hardly any checks on the block payload. // Instead, the FollowerState relies on the consensus nodes to run the full @@ -45,7 +41,6 @@ type FollowerState struct { tracer module.Tracer consumer protocol.Consumer blockTimer protocol.BlockTimer - cfg Config } // MutableState implements a mutable protocol state. When extending the @@ -73,7 +68,6 @@ func NewFollowerState( tracer: tracer, consumer: consumer, blockTimer: blockTimer, - cfg: DefaultConfig(), } return followerState, nil } @@ -159,8 +153,7 @@ func (m *MutableState) Extend(ctx context.Context, candidate *flow.Block) error return fmt.Errorf("payload receipt(s) not compliant with chain state: %w", err) } - // check if the seals in the payload is a valid extension of the finalized - // state + // check if the seals in the payload is a valid extension of the finalized state lastSeal, err := m.sealExtend(ctx, candidate) if err != nil { return fmt.Errorf("payload seal(s) not compliant with chain state: %w", err) @@ -198,6 +191,10 @@ func (m *FollowerState) headerExtend(candidate *flow.Block) error { return state.NewInvalidExtensionErrorf("candidate built for invalid chain (candidate: %s, parent: %s)", header.ChainID, parent.ChainID) } + if header.ParentView != parent.View { + return state.NewInvalidExtensionErrorf("candidate build with inconsistent parent view (candidate: %d, parent %d)", + header.ParentView, parent.View) + } if header.Height != parent.Height+1 { return state.NewInvalidExtensionErrorf("candidate built with invalid height (candidate: %d, parent: %d)", header.Height, parent.Height) @@ -266,20 +263,12 @@ func (m *MutableState) guaranteeExtend(ctx context.Context, candidate *flow.Bloc // we only look as far back for duplicates as the transaction expiry limit; // if a guarantee was included before that, we will disqualify it on the // basis of the reference block anyway - limit := header.Height - m.cfg.transactionExpiry + limit := header.Height - flow.DefaultTransactionExpiry if limit > header.Height { // overflow check limit = 0 } - - // look up the root height so we don't look too far back - // initially this is the genesis block height (aka 0). - var rootHeight uint64 - err := m.db.View(operation.RetrieveRootHeight(&rootHeight)) - if err != nil { - return fmt.Errorf("could not retrieve root block height: %w", err) - } - if limit < rootHeight { - limit = rootHeight + if limit < m.sporkRootBlockHeight { + limit = m.sporkRootBlockHeight } // build a list of all previously used guarantees on this part of the chain @@ -331,7 +320,7 @@ func (m *MutableState) guaranteeExtend(ctx context.Context, candidate *flow.Bloc _, err = protocol.FindGuarantors(m, guarantee) if err != nil { if signature.IsInvalidSignerIndicesError(err) || - errors.Is(err, protocol.ErrEpochNotCommitted) || + errors.Is(err, protocol.ErrNextEpochNotCommitted) || errors.Is(err, protocol.ErrClusterNotFound) { return state.NewInvalidExtensionErrorf("guarantee %v contains invalid guarantors: %w", guarantee.ID(), err) } @@ -390,6 +379,7 @@ func (m *MutableState) receiptExtend(ctx context.Context, candidate *flow.Block) // 95 (sealed) <- 96 <- 97 (finalized) <- 98 <- 99 <- 100 // Now, if block 101 is extending block 100, and its payload has a seal for 96, then it will // be the last sealed for block 101. +// No errors are expected during normal operation. func (m *FollowerState) lastSealed(candidate *flow.Block) (*flow.Seal, error) { header := candidate.Header payload := candidate.Payload @@ -400,52 +390,52 @@ func (m *FollowerState) lastSealed(candidate *flow.Block) (*flow.Seal, error) { return nil, fmt.Errorf("could not retrieve parent seal (%x): %w", header.ParentID, err) } - // if the payload of the block has seals, then the last seal is the seal for the highest - // block - if len(payload.Seals) > 0 { - var highestHeader *flow.Header - for i, seal := range payload.Seals { - header, err := m.headers.ByBlockID(seal.BlockID) - if err != nil { - return nil, state.NewInvalidExtensionErrorf("could not retrieve the header %v for seal: %w", seal.BlockID, err) - } + // if the payload of the block has no seals, then the last seal is the seal for the highest block + if len(payload.Seals) == 0 { + return last, nil + } - if i == 0 || header.Height > highestHeader.Height { - highestHeader = header - last = seal - } + ordered, err := protocol.OrderedSeals(payload, m.headers) + if err != nil { + // all errors are unexpected - differentiation is for clearer error messages + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("ordering seals: candidate payload contains seals for unknown block: %s", err.Error()) } + if errors.Is(err, protocol.ErrDiscontinuousSeals) || errors.Is(err, protocol.ErrMultipleSealsForSameHeight) { + return nil, fmt.Errorf("ordering seals: candidate payload contains invalid seal set: %s", err.Error()) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) } - - return last, nil + return ordered[len(ordered)-1], nil } -// insert stores the candidate block in the data base. The -// `candidate` block _must be valid_ (otherwise, the state will be corrupted). +// insert stores the candidate block in the database. +// The `candidate` block _must be valid_ (otherwise, the state will be corrupted). +// dbUpdates contains other database operations which must be applied atomically +// with inserting the block. func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, last *flow.Seal) error { span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorExtendDBInsert) defer span.End() blockID := candidate.ID() + parentID := candidate.Header.ParentID + latestSealID := last.ID() - // SIXTH: epoch transitions and service events - // (i) Determine protocol state for block's _current_ Epoch. - // As we don't have slashing yet, the protocol state is fully - // determined by the Epoch Preparation events. - // (ii) Determine protocol state for block's _next_ Epoch. - // In case any of the payload seals includes system events, - // we need to check if they are valid and must apply them - // to the protocol state as needed. - ops, err := m.handleServiceEvents(candidate) + parent, err := m.headers.ByBlockID(parentID) if err != nil { - return fmt.Errorf("could not handle service events: %w", err) + return fmt.Errorf("could not retrieve block header for %x: %w", parentID, err) } - // FINALLY: Both the header itself and its payload are in compliance with the - // protocol state. We can now store the candidate block, as well as adding - // its final seal to the seal index and initializing its children index. + // apply any state changes from service events sealed by this block's parent + dbUpdates, err := m.handleEpochServiceEvents(candidate) + if err != nil { + return fmt.Errorf("could not process service events: %w", err) + } + // Both the header itself and its payload are in compliance with the protocol state. + // We can now store the candidate block, as well as adding its final seal + // to the seal index and initializing its children index. err = operation.RetryOnConflictTx(m.db, transaction.Update, func(tx *transaction.Tx) error { // insert the block into the database AND cache err := m.blocks.StoreTx(candidate)(tx) @@ -454,7 +444,7 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, last } // index the latest sealed block in this fork - err = transaction.WithTx(operation.IndexLatestSealAtBlock(blockID, last.ID()))(tx) + err = transaction.WithTx(operation.IndexLatestSealAtBlock(blockID, latestSealID))(tx) if err != nil { return fmt.Errorf("could not index candidate seal: %w", err) } @@ -466,7 +456,7 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, last } // apply any optional DB operations from service events - for _, apply := range ops { + for _, apply := range dbUpdates { err := apply(tx) if err != nil { return fmt.Errorf("could not apply operation: %w", err) @@ -475,18 +465,23 @@ func (m *FollowerState) insert(ctx context.Context, candidate *flow.Block, last return nil }) - if err != nil { return fmt.Errorf("could not execute state extension: %w", err) } + // trigger BlockProcessable for parent blocks above root height + if parent.Height > m.rootHeight { + m.consumer.BlockProcessable(parent) + } + return nil } -// Finalize marks the specified block as finalized. This method only -// finalizes one block at a time. Hence, the parent of `blockID` -// has to be the last finalized block. +// Finalize marks the specified block as finalized. +// This method only finalizes one block at a time. +// Hence, the parent of `blockID` has to be the last finalized block. func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) error { + // preliminaries: start tracer and retrieve full block span, _ := m.tracer.StartSpanFromContext(ctx, trace.ProtoStateMutatorFinalize) defer span.End() @@ -496,8 +491,15 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e } header := block.Header - // FIRST: verify that the parent block is the latest finalized block. This - // must be the case, as the `Finalize(..)` method only finalizes one block + // keep track of metrics updates and protocol events to emit: + // * metrics are updated after a successful database update + // * protocol events are emitted atomically with the database update + // TODO deliver protocol events async https://github.com/dapperlabs/flow-go/issues/6317 + var metrics []func() + var events []func() + + // Verify that the parent block is the latest finalized block. + // this must be the case, as the `Finalize` method only finalizes one block // at a time and hence the parent of `blockID` must already be finalized. var finalized uint64 err = m.db.View(operation.RetrieveFinalizedHeight(&finalized)) @@ -513,7 +515,7 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e return fmt.Errorf("can only finalize child of last finalized block") } - // SECOND: We also want to update the last sealed height. Retrieve the block + // We also want to update the last sealed height. Retrieve the block // seal indexed for the block and retrieve the block that was sealed by it. last, err := m.seals.HighestInFork(blockID) if err != nil { @@ -524,16 +526,8 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e return fmt.Errorf("could not retrieve sealed header: %w", err) } - // THIRD: preparing Epoch-Phase-Change service notifications and metrics updates. - // Convention: - // .. <--- P <----- B - // ↑ ↑ - // block sealing service event first block of new - // for epoch-phase transition Epoch phase (e.g. - // (e.g. EpochSetup event) (EpochSetup phase) - // Per convention, service notifications for Epoch-Phase-Changes are emitted, when - // the first block of the new phase (EpochSetup phase) is _finalized_. Meaning - // that the new phase has started. + // We update metrics and emit protocol events for epoch state changes when + // the block corresponding to the state change is finalized epochStatus, err := m.epoch.statuses.ByBlockID(blockID) if err != nil { return fmt.Errorf("could not retrieve epoch state: %w", err) @@ -542,99 +536,51 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e if err != nil { return fmt.Errorf("could not retrieve setup event for current epoch: %w", err) } - parent, err := m.blocks.ByID(header.ParentID) - if err != nil { - return fmt.Errorf("could not get parent (id=%x): %w", header.ParentID, err) - } - - // EECC - check whether the epoch emergency fallback flag has been set - // in the database. If so, skip updating any epoch-related metrics. epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() if err != nil { - return fmt.Errorf("could not check epoch emergency fallback flag: %w", err) + return fmt.Errorf("could not check persisted epoch emergency fallback flag: %w", err) } - // track service event driven metrics and protocol events that should be emitted - var events []func() - for _, seal := range parent.Payload.Seals { - // skip updating epoch-related metrics if EECC is triggered - if epochFallbackTriggered { - break - } - - result, err := m.results.ByID(seal.ResultID) + // if epoch fallback was not previously triggered, check whether this block triggers it + if !epochFallbackTriggered { + epochFallbackTriggered, err = m.epochFallbackTriggeredByFinalizedBlock(header, epochStatus, currentEpochSetup) if err != nil { - return fmt.Errorf("could not retrieve result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + return fmt.Errorf("could not check whether finalized block triggers epoch fallback: %w", err) } - for _, event := range result.ServiceEvents { - switch ev := event.Event.(type) { - case *flow.EpochSetup: - // update current epoch phase - events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseSetup) }) - // track epoch phase transition (staking->setup) - events = append(events, func() { m.consumer.EpochSetupPhaseStarted(ev.Counter-1, header) }) - case *flow.EpochCommit: - // update current epoch phase - events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseCommitted) }) - // track epoch phase transition (setup->committed) - events = append(events, func() { m.consumer.EpochCommittedPhaseStarted(ev.Counter-1, header) }) - // track final view of committed epoch - nextEpochSetup, err := m.epoch.setups.ByID(epochStatus.NextEpoch.SetupID) - if err != nil { - return fmt.Errorf("could not retrieve setup event for next epoch: %w", err) - } - events = append(events, func() { m.metrics.CommittedEpochFinalView(nextEpochSetup.FinalView) }) - default: - return fmt.Errorf("invalid service event type in payload (%T)", event) - } + if epochFallbackTriggered { + // emit the protocol event only the first time epoch fallback is triggered + events = append(events, m.consumer.EpochEmergencyFallbackTriggered) + metrics = append(metrics, m.metrics.EpochEmergencyFallbackTriggered) } } - // FOURTH: preparing Epoch-Change service notifications and metrics updates. - // Convention: - // Service notifications and updating metrics happen when we finalize the _first_ - // block of the new Epoch (same convention as for Epoch-Phase-Changes) - // Approach: We retrieve the parent block's epoch information. If this block's view - // exceeds the final view of its parent's current epoch, this block begins the next epoch. - parentBlocksEpoch := m.AtBlockID(header.ParentID).Epochs().Current() - parentEpochFinalView, err := parentBlocksEpoch.FinalView() - if err != nil { - return fmt.Errorf("could not get parent epoch final view: %w", err) - } - - // When this block's view exceeds the parent epoch's final view, this block - // represents the first block of the next epoch. Therefore we update metrics - // related to the epoch transition here. - // - // We skip updating these metrics when EECC has been triggered - if header.View > parentEpochFinalView && !epochFallbackTriggered { - events = append(events, func() { m.consumer.EpochTransition(currentEpochSetup.Counter, header) }) - - // set current epoch counter corresponding to new epoch - events = append(events, func() { m.metrics.CurrentEpochCounter(currentEpochSetup.Counter) }) - // set epoch phase - since we are starting a new epoch we begin in the staking phase - events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseStaking) }) - // set current epoch view values - events = append( - events, - func() { m.metrics.CurrentEpochFinalView(currentEpochSetup.FinalView) }, - func() { m.metrics.CurrentDKGPhase1FinalView(currentEpochSetup.DKGPhase1FinalView) }, - func() { m.metrics.CurrentDKGPhase2FinalView(currentEpochSetup.DKGPhase2FinalView) }, - func() { m.metrics.CurrentDKGPhase3FinalView(currentEpochSetup.DKGPhase3FinalView) }, - ) - } + // Determine metric updates and protocol events related to epoch phase + // changes and epoch transitions. + // If epoch emergency fallback is triggered, the current epoch continues until + // the next spork - so skip these updates. + if !epochFallbackTriggered { + epochPhaseMetrics, epochPhaseEvents, err := m.epochPhaseMetricsAndEventsOnBlockFinalized(header, epochStatus) + if err != nil { + return fmt.Errorf("could not determine epoch phase metrics/events for finalized block: %w", err) + } + metrics = append(metrics, epochPhaseMetrics...) + events = append(events, epochPhaseEvents...) - // if EECC is triggered, update metric - if epochFallbackTriggered { - m.metrics.EpochEmergencyFallbackTriggered() + epochTransitionMetrics, epochTransitionEvents, err := m.epochTransitionMetricsAndEventsOnBlockFinalized(header, currentEpochSetup) + if err != nil { + return fmt.Errorf("could not determine epoch transition metrics/events for finalized block: %w", err) + } + metrics = append(metrics, epochTransitionMetrics...) + events = append(events, epochTransitionEvents...) } - // FIFTH: Persist updates in database + // Persist updates in database // * Add this block to the height-indexed set of finalized blocks. // * Update the largest finalized height to this block's height. // * Update the largest height of sealed and finalized block. // This value could actually stay the same if it has no seals in // its payload, in which case the parent's seal is the same. + // * set the epoch fallback flag, if it is triggered err = operation.RetryOnConflict(m.db.Update, func(tx *badger.Txn) error { err = operation.IndexBlockHeight(header.Height, blockID)(tx) if err != nil { @@ -648,6 +594,12 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e if err != nil { return fmt.Errorf("could not update sealed height: %w", err) } + if epochFallbackTriggered { + err = operation.SetEpochEmergencyFallbackTriggered(blockID)(tx) + if err != nil { + return fmt.Errorf("could not set epoch fallback flag: %w", err) + } + } // When a block is finalized, we commit the result for each seal it contains. The sealing logic // guarantees that only a single, continuous execution fork is sealed. Here, we index for @@ -659,23 +611,24 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e } } - // emit protocol events within the scope of the Badger transaction to - // guarantee at-least-once delivery - m.consumer.BlockFinalized(header) - for _, emit := range events { - emit() - } return nil }) if err != nil { - return fmt.Errorf("could not execute finalization: %w", err) + return fmt.Errorf("could not persist finalization operations for block (%x): %w", blockID, err) } - // FINALLY: update metrics + // Emit protocol events after database transaction succeeds. Event delivery is guaranteed, + // _except_ in case of a crash. Hence, when recovering from a crash, consumers need to deduce + // from the state whether they have missed events and re-execute the respective actions. + m.consumer.BlockFinalized(header) + for _, emit := range events { + emit() + } + + // update sealed/finalized block metrics m.metrics.FinalizedHeight(header.Height) m.metrics.SealedHeight(sealed.Height) m.metrics.BlockFinalized(block) - for _, seal := range block.Payload.Seals { sealedBlock, err := m.blocks.ByID(seal.BlockID) if err != nil { @@ -684,63 +637,230 @@ func (m *FollowerState) Finalize(ctx context.Context, blockID flow.Identifier) e m.metrics.BlockSealed(sealedBlock) } + // apply all queued metrics + for _, updateMetric := range metrics { + updateMetric() + } + return nil } -// epochStatus computes the EpochStatus for the given block -// BEFORE applying the block payload itself +// epochFallbackTriggeredByFinalizedBlock checks whether finalizing the input block +// would trigger epoch emergency fallback mode. In particular, we trigger epoch +// fallback mode while finalizing block B in either of the following cases: +// 1. B is the head of a fork in which epoch fallback was tentatively triggered, +// due to incorporating an invalid service event. +// 2. (a) B is the first finalized block with view greater than or equal to the epoch +// commitment deadline for the current epoch AND +// (b) the next epoch has not been committed as of B. +// +// This function should only be called when epoch fallback *has not already been triggered*. +// See protocol.Params for more details on the epoch commitment deadline. +// +// No errors are expected during normal operation. +func (m *FollowerState) epochFallbackTriggeredByFinalizedBlock(block *flow.Header, epochStatus *flow.EpochStatus, currentEpochSetup *flow.EpochSetup) (bool, error) { + // 1. Epoch fallback is tentatively triggered on this fork + if epochStatus.InvalidServiceEventIncorporated { + return true, nil + } + + // 2.(a) determine whether block B is past the epoch commitment deadline + safetyThreshold, err := m.Params().EpochCommitSafetyThreshold() + if err != nil { + return false, fmt.Errorf("could not get epoch commit safety threshold: %w", err) + } + blockExceedsDeadline := block.View+safetyThreshold >= currentEpochSetup.FinalView + + // 2.(b) determine whether the next epoch is committed w.r.t. block B + currentEpochPhase, err := epochStatus.Phase() + if err != nil { + return false, fmt.Errorf("could not get current epoch phase: %w", err) + } + isNextEpochCommitted := currentEpochPhase == flow.EpochPhaseCommitted + + blockTriggersEpochFallback := blockExceedsDeadline && !isNextEpochCommitted + return blockTriggersEpochFallback, nil +} + +// epochTransitionMetricsAndEventsOnBlockFinalized determines metrics to update +// and protocol events to emit, if this block is the first of a new epoch. +// +// Protocol events and updating metrics happen when we finalize the _first_ +// block of the new Epoch (same convention as for Epoch-Phase-Changes). +// Approach: We retrieve the parent block's epoch information. +// When this block's view exceeds the parent epoch's final view, this block +// represents the first block of the next epoch. Therefore we update metrics +// related to the epoch transition here. +// +// This function should only be called when epoch fallback *has not already been triggered*. +// No errors are expected during normal operation. +func (m *FollowerState) epochTransitionMetricsAndEventsOnBlockFinalized(block *flow.Header, currentEpochSetup *flow.EpochSetup) ( + metrics []func(), + events []func(), + err error, +) { + + parentBlocksEpoch := m.AtBlockID(block.ParentID).Epochs().Current() + parentEpochFinalView, err := parentBlocksEpoch.FinalView() + if err != nil { + return nil, nil, fmt.Errorf("could not get parent epoch final view: %w", err) + } + + if block.View > parentEpochFinalView { + // sanity check: new currentEpochSetup extends parent epoch + parentEpochCounter, err := parentBlocksEpoch.Counter() + if err != nil { + return nil, nil, fmt.Errorf("could not retrieve parent epoch counter: %w", err) + } + if parentEpochCounter+1 != currentEpochSetup.Counter { + return nil, nil, fmt.Errorf("sanity check failed: counter for new current epoch is not consecutive with parent epoch (expected %d+1 = %d)", + parentEpochCounter, currentEpochSetup.Counter) + } + + events = append(events, func() { m.consumer.EpochTransition(currentEpochSetup.Counter, block) }) + + // set current epoch counter corresponding to new epoch + metrics = append(metrics, func() { m.metrics.CurrentEpochCounter(currentEpochSetup.Counter) }) + // set epoch phase - since we are starting a new epoch we begin in the staking phase + metrics = append(metrics, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseStaking) }) + // set current epoch view values + metrics = append( + metrics, + func() { m.metrics.CurrentEpochFinalView(currentEpochSetup.FinalView) }, + func() { m.metrics.CurrentDKGPhase1FinalView(currentEpochSetup.DKGPhase1FinalView) }, + func() { m.metrics.CurrentDKGPhase2FinalView(currentEpochSetup.DKGPhase2FinalView) }, + func() { m.metrics.CurrentDKGPhase3FinalView(currentEpochSetup.DKGPhase3FinalView) }, + ) + } + + return +} + +// epochPhaseMetricsAndEventsOnBlockFinalized determines metrics to update +// and protocol events to emit, if this block is the first of a new epoch phase. +// +// Protocol events and metric updates happen when we finalize the block at +// which a service event causing an epoch phase change comes into effect. +// See handleEpochServiceEvents for details. +// +// Convention: +// +// A <-- ... <-- P(Seal_A) <----- B +// ↑ ↑ +// block sealing service event first block of new Epoch phase +// for epoch-phase transition (e.g. EpochSetup phase) +// (e.g. EpochSetup event) +// +// Per convention, protocol events for epoch phase changes are emitted when +// the first block of the new phase (eg. EpochSetup phase) is _finalized_. +// Meaning that the new phase has started. +// +// This function should only be called when epoch fallback *has not already been triggered*. +// No errors are expected during normal operation. +func (m *FollowerState) epochPhaseMetricsAndEventsOnBlockFinalized(block *flow.Header, epochStatus *flow.EpochStatus) ( + metrics []func(), + events []func(), + err error, +) { + + parent, err := m.blocks.ByID(block.ParentID) + if err != nil { + return nil, nil, fmt.Errorf("could not get parent (id=%x): %w", block.ParentID, err) + } + + // track service event driven metrics and protocol events that should be emitted + for _, seal := range parent.Payload.Seals { + + result, err := m.results.ByID(seal.ResultID) + if err != nil { + return nil, nil, fmt.Errorf("could not retrieve result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) + } + for _, event := range result.ServiceEvents { + switch ev := event.Event.(type) { + case *flow.EpochSetup: + // update current epoch phase + events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseSetup) }) + // track epoch phase transition (staking->setup) + events = append(events, func() { m.consumer.EpochSetupPhaseStarted(ev.Counter-1, block) }) + case *flow.EpochCommit: + // update current epoch phase + events = append(events, func() { m.metrics.CurrentEpochPhase(flow.EpochPhaseCommitted) }) + // track epoch phase transition (setup->committed) + events = append(events, func() { m.consumer.EpochCommittedPhaseStarted(ev.Counter-1, block) }) + // track final view of committed epoch + nextEpochSetup, err := m.epoch.setups.ByID(epochStatus.NextEpoch.SetupID) + if err != nil { + return nil, nil, fmt.Errorf("could not retrieve setup event for next epoch: %w", err) + } + events = append(events, func() { m.metrics.CommittedEpochFinalView(nextEpochSetup.FinalView) }) + default: + return nil, nil, fmt.Errorf("invalid service event type in payload (%T)", event) + } + } + } + + return +} + +// epochStatus computes the EpochStatus for the given block *before* applying +// any service event state changes which come into effect with this block. +// // Specifically, we must determine whether block is the first block of a new // epoch in its respective fork. We do this by comparing the block's view to // the Epoch data from its parent. If the block's view is _larger_ than the // final View of the parent's epoch, the block starts a new Epoch. -// - case (a): block is in same Epoch as parent. -// the parent's EpochStatus.CurrentEpoch also applies for the current block -// - case (b): block starts new Epoch in its respective fork. -// the parent's EpochStatus.NextEpoch is the current block's EpochStatus.CurrentEpoch // -// As the parent was a valid extension of the chain, by induction, the parent satisfies all -// consistency requirements of the protocol. +// Possible outcomes: +// 1. Block is in same Epoch as parent (block.View < epoch.FinalView) +// -> the parent's EpochStatus.CurrentEpoch also applies for the current block +// 2. Block enters the next Epoch (block.View ≥ epoch.FinalView) +// a) HAPPY PATH: Epoch fallback is not triggered, we enter the next epoch: +// -> the parent's EpochStatus.NextEpoch is the current block's EpochStatus.CurrentEpoch +// b) FALLBACK PATH: Epoch fallback is triggered, we continue the current epoch: +// -> the parent's EpochStatus.CurrentEpoch also applies for the current block // -// Returns: -// - errIncompleteEpochConfiguration if the epoch has ended before processing -// both an EpochSetup and EpochCommit event; so the new epoch can't be constructed. -func (m *FollowerState) epochStatus(block *flow.Header) (*flow.EpochStatus, error) { - +// As the parent was a valid extension of the chain, by induction, the parent +// satisfies all consistency requirements of the protocol. +// +// Returns the EpochStatus for the input block. +// No error returns are expected under normal operations +func (m *FollowerState) epochStatus(block *flow.Header, epochFallbackTriggered bool) (*flow.EpochStatus, error) { parentStatus, err := m.epoch.statuses.ByBlockID(block.ParentID) if err != nil { return nil, fmt.Errorf("could not retrieve epoch state for parent: %w", err) } - - // Retrieve EpochSetup and EpochCommit event for parent block's Epoch parentSetup, err := m.epoch.setups.ByID(parentStatus.CurrentEpoch.SetupID) if err != nil { return nil, fmt.Errorf("could not retrieve EpochSetup event for parent: %w", err) } - if parentSetup.FinalView < block.View { // first block of a new epoch - // sanity check: parent's epoch Preparation should be completed and have EpochSetup and EpochCommit events - if parentStatus.NextEpoch.SetupID == flow.ZeroID { - return nil, fmt.Errorf("missing setup event for starting next epoch: %w", errIncompleteEpochConfiguration) - } - if parentStatus.NextEpoch.CommitID == flow.ZeroID { - return nil, fmt.Errorf("missing commit event for starting next epoch: %w", errIncompleteEpochConfiguration) - } - status, err := flow.NewEpochStatus( - parentStatus.CurrentEpoch.SetupID, parentStatus.CurrentEpoch.CommitID, - parentStatus.NextEpoch.SetupID, parentStatus.NextEpoch.CommitID, - flow.ZeroID, flow.ZeroID, - ) - return status, err + // Case 1 or 2b (still in parent block's epoch or epoch fallback triggered): + if block.View <= parentSetup.FinalView || epochFallbackTriggered { + // IMPORTANT: copy the status to avoid modifying the parent status in the cache + return parentStatus.Copy(), nil + } + + // Case 2a (first block of new epoch): + // sanity check: parent's epoch Preparation should be completed and have EpochSetup and EpochCommit events + if parentStatus.NextEpoch.SetupID == flow.ZeroID { + return nil, fmt.Errorf("missing setup event for starting next epoch") + } + if parentStatus.NextEpoch.CommitID == flow.ZeroID { + return nil, fmt.Errorf("missing commit event for starting next epoch") } + epochStatus, err := flow.NewEpochStatus( + parentStatus.CurrentEpoch.SetupID, parentStatus.CurrentEpoch.CommitID, + parentStatus.NextEpoch.SetupID, parentStatus.NextEpoch.CommitID, + flow.ZeroID, flow.ZeroID, + ) + return epochStatus, err - // Block is in the same epoch as its parent, re-use the same epoch status - // IMPORTANT: copy the status to avoid modifying the parent status in the cache - currentStatus := parentStatus.Copy() - return currentStatus, err } -// handleServiceEvents handles applying state changes which occur as a result -// of service events being included in a block payload. +// handleEpochServiceEvents handles applying state changes which occur as a result +// of service events being included in a block payload: +// * inserting incorporated service events +// * updating EpochStatus for the candidate block // // Consider a chain where a service event is emitted during execution of block A. // Block B contains a receipt for A. Block C contains a seal for block A. Block @@ -755,74 +875,65 @@ func (m *FollowerState) epochStatus(block *flow.Header) (*flow.EpochStatus, erro // with MarkValid. Because of this, any change to the protocol state introduced // by a service event emitted in A would only become visible when querying D or // later (D's children). +// TODO(active-pacemaker) update docs here (remove reference to MarkValid) https://github.com/dapperlabs/flow-go/issues/6254 // // This method will only apply service-event-induced state changes when the // input block has the form of block D (ie. has a parent, which contains a seal // for a block in which a service event was emitted). // -// If the service events are valid, or there are no service events, this method -// returns a slice of Badger operations to apply while storing the block. This -// includes an operation to index the epoch status for every block, and -// operations to insert service events for blocks that include them. -// // Return values: -// - ops: pending database operations to persist this processing step -// - error: no errors expected during normal operations -func (m *FollowerState) handleServiceEvents(block *flow.Block) ([]func(*transaction.Tx) error, error) { - var ops []func(*transaction.Tx) error - blockID := block.ID() - - // Determine epoch status for block's CURRENT epoch. - // - // This yields the tentative protocol state BEFORE applying the block payload. - // As we don't have slashing yet, there is nothing in the payload which could - // modify the protocol state for the current epoch. - - epochStatus, err := m.epochStatus(block.Header) - if errors.Is(err, errIncompleteEpochConfiguration) { - // TMP: EMERGENCY EPOCH CHAIN CONTINUATION - // - // We are proposing or processing the first block of the next epoch, - // but that epoch has not been setup. Rather than returning an error - // which prevents further block production, we store the block with - // the same epoch status as its parent, resulting in it being considered - // by the protocol state to fall in the same epoch as its parent. - // - // CAUTION: this is inconsistent with the FinalView value specified in the epoch. - parentStatus, err := m.epoch.statuses.ByBlockID(block.Header.ParentID) - if err != nil { - return nil, fmt.Errorf("internal error constructing EECC from parent's epoch status: %w", err) - } - ops = append(ops, m.epoch.statuses.StoreTx(blockID, parentStatus.Copy())) - ops = append(ops, transaction.WithTx(operation.SetEpochEmergencyFallbackTriggered(blockID))) - ops = append(ops, func(tx *transaction.Tx) error { - tx.OnSucceed(m.metrics.EpochEmergencyFallbackTriggered) - return nil - }) - return ops, nil - } else if err != nil { - return nil, fmt.Errorf("could not determine epoch status: %w", err) +// - dbUpdates - If the service events are valid, or there are no service events, +// this method returns a slice of Badger operations to apply while storing the block. +// This includes an operation to index the epoch status for every block, and +// operations to insert service events for blocks that include them. +// +// No errors are expected during normal operation. +func (m *FollowerState) handleEpochServiceEvents(candidate *flow.Block) (dbUpdates []func(*transaction.Tx) error, err error) { + epochFallbackTriggered, err := m.isEpochEmergencyFallbackTriggered() + if err != nil { + return nil, fmt.Errorf("could not retrieve epoch fallback status: %w", err) + } + epochStatus, err := m.epochStatus(candidate.Header, epochFallbackTriggered) + if err != nil { + return nil, fmt.Errorf("could not determine epoch status for candidate block: %w", err) } - activeSetup, err := m.epoch.setups.ByID(epochStatus.CurrentEpoch.SetupID) if err != nil { return nil, fmt.Errorf("could not retrieve current epoch setup event: %w", err) } - // we will apply service events from blocks which are sealed by this block's PARENT - parent, err := m.blocks.ByID(block.Header.ParentID) - if err != nil { - return nil, fmt.Errorf("could not get parent (id=%x): %w", block.Header.ParentID, err) + // always persist the candidate's epoch status + // note: We are scheduling the operation to store the Epoch status using the _pointer_ variable `epochStatus`. + // The struct `epochStatus` points to will still be modified below. + blockID := candidate.ID() + dbUpdates = append(dbUpdates, m.epoch.statuses.StoreTx(blockID, epochStatus)) + + // never process service events after epoch fallback is triggered + if epochStatus.InvalidServiceEventIncorporated || epochFallbackTriggered { + return dbUpdates, nil } - // The payload might contain epoch preparation service events for the next + // We apply service events from blocks which are sealed by this block's PARENT. + // The parent's payload might contain epoch preparation service events for the next // epoch. In this case, we need to update the tentative protocol state. // We need to validate whether all information is available in the protocol // state to go to the next epoch when needed. In cases where there is a bug // in the smart contract, it could be that this happens too late and the // chain finalization should halt. -SealLoop: - for _, seal := range parent.Payload.Seals { + parent, err := m.blocks.ByID(candidate.Header.ParentID) + if err != nil { + return nil, fmt.Errorf("could not get parent (id=%x): %w", candidate.Header.ParentID, err) + } + + // block payload may not specify seals in order, so order them by block height before processing + orderedSeals, err := protocol.OrderedSeals(parent.Payload, m.headers) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("ordering seals: parent payload contains seals for unknown block: %s", err.Error()) + } + return nil, fmt.Errorf("unexpected error ordering seals: %w", err) + } + for _, seal := range orderedSeals { result, err := m.results.ByID(seal.ResultID) if err != nil { return nil, fmt.Errorf("could not get result (id=%x) for seal (id=%x): %w", seal.ResultID, seal.ID(), err) @@ -832,117 +943,63 @@ SealLoop: switch ev := event.Event.(type) { case *flow.EpochSetup: - // validate the service event err := isValidExtendingEpochSetup(ev, activeSetup, epochStatus) - if protocol.IsInvalidServiceEventError(err) { - // EECC - we have observed an invalid service event, which is - // an unrecoverable failure. Flag this in the DB and exit - ops = append(ops, transaction.WithTx(operation.SetEpochEmergencyFallbackTriggered(blockID))) - break SealLoop + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochSetup service event: %w", err) } // prevents multiple setup events for same Epoch (including multiple setup events in payload of same block) epochStatus.NextEpoch.SetupID = ev.ID() // we'll insert the setup event when we insert the block - ops = append(ops, m.epoch.setups.StoreTx(ev)) + dbUpdates = append(dbUpdates, m.epoch.setups.StoreTx(ev)) case *flow.EpochCommit: + // if we receive an EpochCommit event, we must have already observed an EpochSetup event + // => otherwise, we have observed an EpochCommit without corresponding EpochSetup, which triggers epoch fallback mode + if epochStatus.NextEpoch.SetupID == flow.ZeroID { + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + // if we have observed an EpochSetup event, we must be able to retrieve it from the database + // => otherwise, this is a symptom of bug or data corruption since this component sets the SetupID field extendingSetup, err := m.epoch.setups.ByID(epochStatus.NextEpoch.SetupID) if err != nil { - return nil, state.NewInvalidExtensionErrorf("could not retrieve next epoch setup: %s", err) + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("unexpected failure to retrieve EpochSetup (id=%x) stored in EpochStatus for block %x: %w", + epochStatus.NextEpoch.SetupID, blockID, err) + } + return nil, fmt.Errorf("unexpected error retrieving next epoch setup: %w", err) } + // validate the service event err = isValidExtendingEpochCommit(ev, extendingSetup, activeSetup, epochStatus) - if protocol.IsInvalidServiceEventError(err) { - // EECC - we have observed an invalid service event, which is - // an unrecoverable failure. Flag this in the DB and exit - ops = append(ops, transaction.WithTx(operation.SetEpochEmergencyFallbackTriggered(blockID))) - break SealLoop + if err != nil { + if protocol.IsInvalidServiceEventError(err) { + // we have observed an invalid service event, which triggers epoch fallback mode + epochStatus.InvalidServiceEventIncorporated = true + return dbUpdates, nil + } + return nil, fmt.Errorf("unexpected error validating EpochCommit service event: %w", err) } // prevents multiple setup events for same Epoch (including multiple setup events in payload of same block) epochStatus.NextEpoch.CommitID = ev.ID() // we'll insert the commit event when we insert the block - ops = append(ops, m.epoch.commits.StoreTx(ev)) + dbUpdates = append(dbUpdates, m.epoch.commits.StoreTx(ev)) default: - return nil, fmt.Errorf("invalid service event type: %s", event.Type) + return nil, fmt.Errorf("invalid service event type (type_name=%s, go_type=%T)", event.Type, ev) } } } - - // we always index the epoch status, even when there are no service events - ops = append(ops, m.epoch.statuses.StoreTx(block.ID(), epochStatus)) - - return ops, nil -} - -// MarkValid marks the block as valid in protocol state, and triggers -// `BlockProcessable` event to notify that its parent block is processable. -// -// Why is the parent block processable, not the block itself? -// because a block having a child block means it has been verified -// by the majority of consensus participants. -// Hence, if a block has passed the header validity check, its parent block -// must have passed both the header validity check and the body validity check. -// So that consensus followers can skip the block body validity checks and wait -// for its child to arrive, and if the child passes the header validity check, it means -// the consensus participants have done a complete check on its parent block, -// so consensus followers can trust consensus nodes did the right job, and start -// processing the parent block. -// -// NOTE: since a parent can have multiple children, `BlockProcessable` event -// could be triggered multiple times for the same block. -// NOTE: BlockProcessable should not be blocking, otherwise, it will block the follower -func (m *FollowerState) MarkValid(blockID flow.Identifier) error { - header, err := m.headers.ByBlockID(blockID) - if err != nil { - return fmt.Errorf("could not retrieve block header for %x: %w", blockID, err) - } - parentID := header.ParentID - var isParentValid bool - err = m.db.View(operation.RetrieveBlockValidity(parentID, &isParentValid)) - if err != nil { - return fmt.Errorf("could not retrieve validity of parent block (%x): %w", parentID, err) - } - if !isParentValid { - return fmt.Errorf("can only mark block as valid whose parent is valid") - } - - parent, err := m.headers.ByBlockID(parentID) - if err != nil { - return fmt.Errorf("could not retrieve block header for %x: %w", parentID, err) - } - // root blocks and blocks below the root block are considered as "processed", - // so we don't want to trigger `BlockProcessable` event for them. - var rootHeight uint64 - err = m.db.View(operation.RetrieveRootHeight(&rootHeight)) - if err != nil { - return fmt.Errorf("could not retrieve root block's height: %w", err) - } - - err = operation.RetryOnConflict(m.db.Update, func(tx *badger.Txn) error { - // insert block validity for this block - err = operation.SkipDuplicates(operation.InsertBlockValidity(blockID, true))(tx) - if err != nil { - return fmt.Errorf("could not insert validity for block (id=%x, height=%d): %w", blockID, header.Height, err) - } - - // trigger BlockProcessable for parent blocks above root height - if parent.Height > rootHeight { - // emit protocol event within the scope of the Badger transaction to - // guarantee at-least-once delivery - m.consumer.BlockProcessable(parent) - } - return nil - }) - if err != nil { - return fmt.Errorf("could not mark block as valid (%x): %w", blockID, err) - } - - return nil + return } diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index 6704fb5d1fa..d836fb30675 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -83,19 +83,21 @@ func TestBootstrapValid(t *testing.T) { }) } +// TestExtendValid tests the happy path of extending the state with a single block. +// * BlockFinalized is emitted when the block is finalized +// * BlockProcessable is emitted when a block's child is inserted func TestExtendValid(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() tracer := trace.NewNoopTracer() headers, _, seals, index, payloads, blocks, setups, commits, statuses, results := storeutil.StorageLayer(t, db) - // create a event consumer to test epoch transition events distributor := events.NewDistributor() - consumer := new(mockprotocol.Consumer) + consumer := mockprotocol.NewConsumer(t) distributor.AddConsumer(consumer) block, result, seal := unittest.BootstrapFixture(participants) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(block.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID())) rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) @@ -106,21 +108,26 @@ func TestExtendValid(t *testing.T) { util.MockReceiptValidator(), util.MockSealValidator(seals)) require.NoError(t, err) - extend := unittest.BlockWithParentFixture(block.Header) - extend.Payload.Guarantees = nil - extend.Header.PayloadHash = extend.Payload.Hash() - - err = fullState.Extend(context.Background(), extend) + // insert block1 on top of the root block + block1 := unittest.BlockWithParentFixture(block.Header) + err = fullState.Extend(context.Background(), block1) require.NoError(t, err) - finalCommit, err := state.Final().Commit() - require.NoError(t, err) - require.Equal(t, seal.FinalState, finalCommit) + // we should not emit BlockProcessable for the root block + consumer.AssertNotCalled(t, "BlockProcessable", block.Header) - consumer.On("BlockFinalized", extend.Header).Once() - err = fullState.Finalize(context.Background(), extend.ID()) - require.NoError(t, err) - consumer.AssertExpectations(t) + t.Run("BlockFinalized event should be emitted when block1 is finalized", func(t *testing.T) { + consumer.On("BlockFinalized", block1.Header).Once() + err := fullState.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + }) + + t.Run("BlockProcessable event should be emitted when any child of block1 is inserted", func(t *testing.T) { + block2 := unittest.BlockWithParentFixture(block1.Header) + consumer.On("BlockProcessable", block1.Header).Once() + err := fullState.Extend(context.Background(), block2) + require.NoError(t, err) + }) }) } @@ -340,6 +347,7 @@ func TestExtendHeightTooSmall(t *testing.T) { extend.Header.Height = 1 extend.Header.View = 1 extend.Header.ParentID = head.ID() + extend.Header.ParentView = head.View err = state.Extend(context.Background(), &extend) require.NoError(t, err) @@ -377,6 +385,26 @@ func TestExtendHeightTooLarge(t *testing.T) { }) } +// TestExtendInconsistentParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent +// with view of block referred by ParentID. +func TestExtendInconsistentParentView(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + + head, err := rootSnapshot.Head() + require.NoError(t, err) + + block := unittest.BlockWithParentFixture(head) + block.SetPayload(flow.EmptyPayload()) + // set an invalid parent view + block.Header.ParentView++ + + err = state.Extend(context.Background(), block) + require.Error(t, err) + require.True(t, st.IsInvalidExtensionError(err)) + }) +} + func TestExtendBlockNotConnected(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { @@ -467,7 +495,7 @@ func TestExtendReceiptsNotSorted(t *testing.T) { } func TestExtendReceiptsInvalid(t *testing.T) { - validator := &mockmodule.ReceiptValidator{} + validator := mockmodule.NewReceiptValidator(t) rootSnapshot := unittest.RootSnapshotFixture(participants) util.RunWithFullProtocolStateAndValidator(t, rootSnapshot, validator, func(db *badger.DB, state *protocol.MutableState) { @@ -497,7 +525,6 @@ func TestExtendReceiptsInvalid(t *testing.T) { err = state.Extend(context.Background(), block3) require.Error(t, err) require.True(t, st.IsInvalidExtensionError(err), err) - validator.AssertExpectations(t) }) } @@ -566,14 +593,15 @@ func TestExtendReceiptsValid(t *testing.T) { // B9 is the first block of the NEXT epoch. func TestExtendEpochTransitionValid(t *testing.T) { // create a event consumer to test epoch transition events - consumer := new(mockprotocol.Consumer) + consumer := mockprotocol.NewConsumer(t) consumer.On("BlockFinalized", mock.Anything) + consumer.On("BlockProcessable", mock.Anything) rootSnapshot := unittest.RootSnapshotFixture(participants) unittest.RunWithBadgerDB(t, func(db *badger.DB) { // set up state and mock ComplianceMetrics object - metrics := new(mockmodule.ComplianceMetrics) + metrics := mockmodule.NewComplianceMetrics(t) metrics.On("BlockSealed", mock.Anything) metrics.On("SealedHeight", mock.Anything) metrics.On("FinalizedHeight", mock.Anything) @@ -832,15 +860,6 @@ func TestExtendEpochTransitionValid(t *testing.T) { require.NoError(t, err) err = state.Finalize(context.Background(), block9.ID()) require.NoError(t, err) - consumer.AssertCalled(t, "EpochTransition", epoch2Setup.Counter, block9.Header) - metrics.AssertCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) - metrics.AssertCalled(t, "CurrentEpochPhase", flow.EpochPhaseStaking) - metrics.AssertCalled(t, "CurrentEpochFinalView", epoch2Setup.FinalView) - metrics.AssertCalled(t, "CurrentDKGPhase1FinalView", epoch2Setup.DKGPhase1FinalView) - metrics.AssertCalled(t, "CurrentDKGPhase2FinalView", epoch2Setup.DKGPhase2FinalView) - metrics.AssertCalled(t, "CurrentDKGPhase3FinalView", epoch2Setup.DKGPhase3FinalView) - - metrics.AssertExpectations(t) }) } @@ -1057,10 +1076,19 @@ func TestExtendDuplicateEpochEvents(t *testing.T) { }) } -// extending protocol state with an invalid epoch setup service event should cause an error +// TestExtendEpochSetupInvalid tests that incorporating an invalid EpochSetup +// service event should trigger epoch fallback when the fork is finalized. func TestExtendEpochSetupInvalid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + + // setupState initializes the protocol state for a test case + // * creates and finalizes a new block for the first seal to reference + // * creates a factory method for test cases to generated valid EpochSetup events + setupState := func(t *testing.T, db *badger.DB, state *protocol.MutableState) ( + *flow.Block, + func(...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal), + ) { + head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1069,8 +1097,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) block1.SetPayload(flow.EmptyPayload()) - err = state.Extend(context.Background(), block1) - require.NoError(t, err) + unittest.InsertAndFinalize(t, state, block1) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) @@ -1080,7 +1107,7 @@ func TestExtendEpochSetupInvalid(t *testing.T) { // this function will return a VALID setup event and seal, we will modify // in different ways in each test case - createSetup := func(opts ...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { + createSetupEvent := func(opts ...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { setup := unittest.EpochSetupFixture( unittest.WithParticipants(epoch2Participants), unittest.SetupWithCounter(epoch1Setup.Counter+1), @@ -1096,55 +1123,105 @@ func TestExtendEpochSetupInvalid(t *testing.T) { return setup, receipt, seal } - // expect a setup event with wrong counter to trigger EECC without error - t.Run("wrong counter (EECC)", func(t *testing.T) { + return block1, createSetupEvent + } + + // expect a setup event with wrong counter to trigger EECC without error + t.Run("wrong counter (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup := setupState(t, db, state) + _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { - setup.Counter = epoch1Setup.Counter + setup.Counter = rand.Uint64() }) - sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) + }) + + // expect a setup event with wrong final view to trigger EECC without error + t.Run("invalid final view (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup := setupState(t, db, state) - // expect a setup event with wrong final view to trigger EECC without error - t.Run("invalid final view (EECC)", func(t *testing.T) { _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.FinalView = block1.Header.View }) - sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) + }) + + // expect a setup event with empty seed to trigger EECC without error + t.Run("empty seed (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup := setupState(t, db, state) - // expect a setup event with empty seed to trigger EECC without error - t.Run("empty seed (EECC)", func(t *testing.T) { _, receipt, seal := createSetup(func(setup *flow.EpochSetup) { setup.RandomSource = nil }) - sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) }) } -// extending protocol state with an invalid epoch commit service event should cause an error +// TestExtendEpochCommitInvalid tests that incorporating an invalid EpochCommit +// service event should trigger epoch fallback when the fork is finalized. func TestExtendEpochCommitInvalid(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + // setupState initializes the protocol state for a test case + // * creates and finalizes a new block for the first seal to reference + // * creates a factory method for test cases to generated valid EpochSetup events + // * creates a factory method for test cases to generated valid EpochCommit events + setupState := func(t *testing.T, state *protocol.MutableState) ( + *flow.Block, + func(*flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal), + func(*flow.Block, ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal), + ) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1153,8 +1230,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) block1.SetPayload(flow.EmptyPayload()) - err = state.Extend(context.Background(), block1) - require.NoError(t, err) + unittest.InsertAndFinalize(t, state, block1) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) @@ -1165,6 +1241,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { epoch2NewParticipant, ).Sort(order.Canonical) + // factory method to create a valid EpochSetup method w.r.t. the generated state createSetup := func(block *flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) { setup := unittest.EpochSetupFixture( unittest.WithParticipants(epoch2Participants), @@ -1179,6 +1256,7 @@ func TestExtendEpochCommitInvalid(t *testing.T) { return setup, receipt, seal } + // factory method to create a valid EpochCommit method w.r.t. the generated state createCommit := func(block *flow.Block, opts ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal) { commit := unittest.EpochCommitFixture( unittest.CommitWithCounter(epoch1Setup.Counter+1), @@ -1193,67 +1271,148 @@ func TestExtendEpochCommitInvalid(t *testing.T) { return commit, receipt, seal } - t.Run("without setup", func(t *testing.T) { + return block1, createSetup, createCommit + } + + t.Run("without setup (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, _, createCommit := setupState(t, state) + _, receipt, seal := createCommit(block1) - sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal) + err := state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) - require.Error(t, err) - require.True(t, st.IsInvalidExtensionError(err), err) + require.NoError(t, err) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) + }) - // seal block 1, in which EpochSetup was emitted - epoch2Setup, setupReceipt, setupSeal := createSetup(block1) - block2 := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + // expect a commit event with wrong counter to trigger EECC without error + t.Run("inconsistent counter (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup, createCommit := setupState(t, state) - // insert a block with a QC for block 2 - block3 := unittest.BlockWithParentFixture(block2) - err = state.Extend(context.Background(), block3) - require.NoError(t, err) + // seal block 1, in which EpochSetup was emitted + epoch2Setup, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) - // expect a commit event with wrong counter to trigger EECC without error - t.Run("inconsistent counter (EECC)", func(t *testing.T) { _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { commit.Counter = epoch2Setup.Counter + 1 }) - sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) + }) + + // expect a commit event with wrong cluster QCs to trigger EECC without error + t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup, createCommit := setupState(t, state) + + // seal block 1, in which EpochSetup was emitted + _, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) - // expect a commit event with wrong cluster QCs to trigger EECC without error - t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) { _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(unittest.QuorumCertificateWithSignerIDsFixture())) }) - sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) + }) + + // expect a commit event with wrong dkg participants to trigger EECC without error + t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) { + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.MutableState) { + block1, createSetup, createCommit := setupState(t, state) + + // seal block 1, in which EpochSetup was emitted + _, setupReceipt, setupSeal := createSetup(block1) + epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal) + err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), epochSetupSealingBlock.ID()) + require.NoError(t, err) + + // insert a block with a QC for block 2 + block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock) + unittest.InsertAndFinalize(t, state, block3) - // expect a commit event with wrong dkg participants to trigger EECC without error - t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) { _, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) { // add an extra dkg key commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey()) }) - sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal) + err = state.Finalize(context.Background(), receiptBlock.ID()) + require.NoError(t, err) + err = state.Finalize(context.Background(), sealingBlock.ID()) + require.NoError(t, err) qcBlock := unittest.BlockWithParentFixture(sealingBlock) err = state.Extend(context.Background(), qcBlock) require.NoError(t, err) - assertEpochEmergencyFallbackTriggered(t, db) + // epoch fallback not triggered before finalization + assertEpochEmergencyFallbackTriggered(t, state, false) + err = state.Finalize(context.Background(), qcBlock.ID()) + require.NoError(t, err) + // epoch fallback triggered after finalization + assertEpochEmergencyFallbackTriggered(t, state, true) }) }) } @@ -1325,100 +1484,94 @@ func TestExtendEpochTransitionWithoutCommit(t *testing.T) { }) } -func TestEmergencyEpochChainContinuation(t *testing.T) { +// TestEmergencyEpochFallback tests that epoch emergency fallback is triggered +// when an epoch fails to be committed before the epoch commitment deadline, +// or when an invalid service event (indicating service account smart contract bug) +// is sealed. +func TestEmergencyEpochFallback(t *testing.T) { - // if we reach the first block of the next epoch before both setup and commit - // service events are finalized, the chain should continue with the previous epoch. + // if we finalize the first block past the epoch commitment deadline while + // in the EpochStaking phase, EECC should be triggered // - // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 - t.Run("epoch transition without commit event - should continue with fallback epoch", func(t *testing.T) { + // Epoch Commitment Deadline + // | Epoch Boundary + // | | + // v v + // ROOT <- B1 <- B2 + t.Run("passed epoch commitment deadline in EpochStaking phase - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - metricsMock := new(mockmodule.ComplianceMetrics) + metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything) - util.RunWithFullProtocolStateAndMetrics(t, rootSnapshot, metricsMock, func(db *badger.DB, state *protocol.MutableState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.MutableState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) - - // add a block for the first seal to reference - block1 := unittest.BlockWithParentFixture(head) - block1.SetPayload(flow.EmptyPayload()) - err = state.Extend(context.Background(), block1) - require.NoError(t, err) - err = state.Finalize(context.Background(), block1.ID()) + safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold() require.NoError(t, err) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) epoch1FinalView := epoch1Setup.FinalView + epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold - // add a participant for the next epoch - epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) - epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) - - // create the epoch setup event for the second epoch - epoch2Setup := unittest.EpochSetupFixture( - unittest.WithParticipants(epoch2Participants), - unittest.SetupWithCounter(epoch1Setup.Counter+1), - unittest.WithFinalView(epoch1FinalView+1000), - unittest.WithFirstView(epoch1FinalView+1), - ) + // finalizing block 1 should trigger EECC + metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() - receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) - receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + // we begin the epoch in the EpochStaking phase and + // block 1 will be the first block on or past the epoch commitment deadline + block1 := unittest.BlockWithParentFixture(head) + block1.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 + err = state.Extend(context.Background(), block1) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block1.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization - // add a block containing a receipt for block 1 + // block 2 will be the first block past the first epoch boundary block2 := unittest.BlockWithParentFixture(block1.Header) - block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + block2.Header.View = epoch1FinalView + 1 err = state.Extend(context.Background(), block2) require.NoError(t, err) err = state.Finalize(context.Background(), block2.ID()) require.NoError(t, err) - // block 3 seals block 1 - block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(flow.Payload{ - Seals: []*flow.Seal{seal1}, - }) - err = state.Extend(context.Background(), block3) - require.NoError(t, err) - - // block 4 will be the first block for epoch 2 - block4 := unittest.BlockWithParentFixture(block3.Header) - block4.Header.View = epoch1Setup.FinalView + 1 - - // inserting block 4 should trigger EECC - metricsMock.On("EpochEmergencyFallbackTriggered").Once() - - err = state.Extend(context.Background(), block4) - require.NoError(t, err) - - assertEpochEmergencyFallbackTriggered(t, db) - - // epoch metrics should not be emitted - metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything) - metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) - metricsMock.AssertExpectations(t) + // since EECC has been triggered, epoch transition metrics should not be updated + metricsMock.AssertNotCalled(t, "EpochTransition", mock.Anything, mock.Anything) + metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch1Setup.Counter+1) }) }) - // if we reach the first block of the next epoch before either setup and commit - // service events are finalized, the chain should continue with the previous epoch. + // if we finalize the first block past the epoch commitment deadline while + // in the EpochSetup phase, EECC should be triggered // - // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 - t.Run("epoch transition without setup event - should continue with fallback epoch", func(t *testing.T) { + // Epoch Commitment Deadline + // | Epoch Boundary + // | | + // v v + // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 <- B5 + t.Run("passed epoch commitment deadline in EpochSetup phase - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - metricsMock := new(mockmodule.ComplianceMetrics) + metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything) - util.RunWithFullProtocolStateAndMetrics(t, rootSnapshot, metricsMock, func(db *badger.DB, state *protocol.MutableState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.MutableState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() require.NoError(t, err) + safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold() + require.NoError(t, err) // add a block for the first seal to reference block1 := unittest.BlockWithParentFixture(head) @@ -1429,7 +1582,24 @@ func TestEmergencyEpochChainContinuation(t *testing.T) { require.NoError(t, err) epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) + epoch1FinalView := epoch1Setup.FinalView + epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold + + // add a participant for the next epoch + epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification)) + epoch2Participants := append(participants, epoch2NewParticipant).Sort(order.Canonical) + + // create the epoch setup event for the second epoch + epoch2Setup := unittest.EpochSetupFixture( + unittest.WithParticipants(epoch2Participants), + unittest.SetupWithCounter(epoch1Setup.Counter+1), + unittest.WithFinalView(epoch1FinalView+1000), + unittest.WithFirstView(epoch1FinalView+1), + ) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} + seal1.ResultID = receipt1.ExecutionResult.ID() // add a block containing a receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) @@ -1446,38 +1616,56 @@ func TestEmergencyEpochChainContinuation(t *testing.T) { }) err = state.Extend(context.Background(), block3) require.NoError(t, err) + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) - // block 4 will be the first block for epoch 2 + // block 4 will be the first block on or past the epoch commitment deadline block4 := unittest.BlockWithParentFixture(block3.Header) - block4.Header.View = epoch1Setup.FinalView + 1 + block4.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2 - // inserting block 4 should trigger EECC + // finalizing block 4 should trigger EECC metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() err = state.Extend(context.Background(), block4) require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block4.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization - assertEpochEmergencyFallbackTriggered(t, db) + // block 5 will be the first block past the first epoch boundary + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.Header.View = epoch1FinalView + 1 + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + err = state.Finalize(context.Background(), block5.ID()) + require.NoError(t, err) - // epoch metrics should not be emitted - metricsMock.AssertNotCalled(t, "EpochTransition", epoch1Setup.Counter+1, mock.Anything) - metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch1Setup.Counter+1) - metricsMock.AssertExpectations(t) + // since EECC has been triggered, epoch transition metrics should not be updated + metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything) + metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) }) }) // if an invalid epoch service event is incorporated, we should: // * not apply the phase transition corresponding to the invalid service event - // * trigger EECC + // * immediately trigger EECC // - // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 - t.Run("epoch transition with invalid service event - should continue with fallback epoch", func(t *testing.T) { + // Epoch Boundary + // | + // v + // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 <- B5 + t.Run("epoch transition with invalid service event - should trigger EECC", func(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) - metricsMock := new(mockmodule.ComplianceMetrics) + metricsMock := mockmodule.NewComplianceMetrics(t) mockMetricsForRootSnapshot(metricsMock, rootSnapshot) + protoEventsMock := mockprotocol.NewConsumer(t) + protoEventsMock.On("BlockFinalized", mock.Anything) + protoEventsMock.On("BlockProcessable", mock.Anything) - util.RunWithFullProtocolStateAndMetrics(t, rootSnapshot, metricsMock, func(db *badger.DB, state *protocol.MutableState) { + util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.MutableState) { head, err := rootSnapshot.Head() require.NoError(t, err) result, _, err := rootSnapshot.SealedResult() @@ -1509,9 +1697,7 @@ func TestEmergencyEpochChainContinuation(t *testing.T) { receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()} - - // incorporating the service event should trigger EECC - metricsMock.On("EpochEmergencyFallbackTriggered").Once() + seal1.ResultID = receipt1.ExecutionResult.ID() // add a block containing a receipt for block 1 block2 := unittest.BlockWithParentFixture(block1.Header) @@ -1528,20 +1714,33 @@ func TestEmergencyEpochChainContinuation(t *testing.T) { }) err = state.Extend(context.Background(), block3) require.NoError(t, err) + err = state.Finalize(context.Background(), block3.ID()) + require.NoError(t, err) - // block 4 will be the first block for epoch 2 - block4 := unittest.BlockWithParentFixture(block3.Header) - block4.Header.View = epoch1Setup.FinalView + 1 + // incorporating the service event should trigger EECC + metricsMock.On("EpochEmergencyFallbackTriggered").Once() + protoEventsMock.On("EpochEmergencyFallbackTriggered").Once() + // block 4 is where the service event state change comes into effect + block4 := unittest.BlockWithParentFixture(block3.Header) err = state.Extend(context.Background(), block4) require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization + err = state.Finalize(context.Background(), block4.ID()) + require.NoError(t, err) + assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization - assertEpochEmergencyFallbackTriggered(t, db) + // block 5 is the first block past the current epoch boundary + block5 := unittest.BlockWithParentFixture(block4.Header) + block5.Header.View = epoch1Setup.FinalView + 1 + err = state.Extend(context.Background(), block5) + require.NoError(t, err) + err = state.Finalize(context.Background(), block5.ID()) + require.NoError(t, err) - // epoch metrics should not be emitted + // since EECC has been triggered, epoch transition metrics should not be updated metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything) metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter) - metricsMock.AssertExpectations(t) }) }) } @@ -1554,8 +1753,9 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { // create a event consumer to test epoch transition events distributor := events.NewDistributor() - consumer := new(mockprotocol.Consumer) + consumer := mockprotocol.NewConsumer(t) distributor.AddConsumer(consumer) + consumer.On("BlockProcessable", mock.Anything) rootSnapshot := unittest.RootSnapshotFixture(participants) @@ -1579,7 +1779,7 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { Seals: []*flow.Seal{block1Seal}, }) - sealValidator := &mockmodule.SealValidator{} + sealValidator := mockmodule.NewSealValidator(t) sealValidator.On("Validate", mock.Anything). Return(func(candidate *flow.Block) *flow.Seal { if candidate.ID() == block3.ID() { @@ -1605,8 +1805,6 @@ func TestExtendInvalidSealsInBlock(t *testing.T) { err = fullState.Extend(context.Background(), block2) require.NoError(t, err) err = fullState.Extend(context.Background(), block3) - - sealValidator.AssertExpectations(t) require.Error(t, err) require.True(t, st.IsInvalidExtensionError(err)) }) @@ -1791,7 +1989,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { block := unittest.BlockWithParentFixture(head) payload := flow.EmptyPayload() payload.Guarantees = []*flow.CollectionGuarantee{ - { + &flow.CollectionGuarantee{ ChainID: cluster.ChainID(), ReferenceBlockID: head.ID(), SignerIndices: validSignerIndices, @@ -1810,7 +2008,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { err = state.Extend(context.Background(), block) require.Error(t, err) require.True(t, signature.IsInvalidSignerIndicesError(err), err) - require.True(t, errors.Is(err, signature.ErrInvalidChecksum), err) + require.ErrorIs(t, err, signature.ErrInvalidChecksum) require.True(t, st.IsInvalidExtensionError(err), err) // now the guarantee has invalid signer indices: the checksum should have 4 bytes, but it only has 1 @@ -1824,7 +2022,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { err = state.Extend(context.Background(), block) require.Error(t, err) require.True(t, signature.IsInvalidSignerIndicesError(err), err) - require.True(t, errors.Is(err, signature.ErrInvalidChecksum), err) + require.ErrorIs(t, err, signature.ErrInvalidChecksum) require.True(t, st.IsInvalidExtensionError(err), err) // let's test even if the checksum is correct, but signer indices is still wrong because the tailing are not 0, @@ -1837,7 +2035,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { err = state.Extend(context.Background(), block) require.Error(t, err) require.True(t, signature.IsInvalidSignerIndicesError(err), err) - require.True(t, errors.Is(err, signature.ErrIllegallyPaddedBitVector), err) + require.ErrorIs(t, err, signature.ErrIllegallyPaddedBitVector) require.True(t, st.IsInvalidExtensionError(err), err) // test imcompatible bit vector length @@ -1846,7 +2044,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { err = state.Extend(context.Background(), block) require.Error(t, err) require.True(t, signature.IsInvalidSignerIndicesError(err), err) - require.True(t, errors.Is(err, signature.ErrIncompatibleBitVectorLength), err) + require.ErrorIs(t, err, signature.ErrIncompatibleBitVectorLength) require.True(t, st.IsInvalidExtensionError(err), err) // revert back to good value @@ -1856,7 +2054,7 @@ func TestExtendInvalidGuarantee(t *testing.T) { payload.Guarantees[0].ReferenceBlockID = flow.ZeroID err = state.Extend(context.Background(), block) require.Error(t, err) - require.True(t, errors.Is(err, storage.ErrNotFound), err) + require.ErrorIs(t, err, storage.ErrNotFound) require.True(t, st.IsInvalidExtensionError(err), err) // revert back to good value @@ -1871,49 +2069,11 @@ func TestExtendInvalidGuarantee(t *testing.T) { payload.Guarantees[0].ChainID = flow.ChainID("some_bad_chain_ID") err = state.Extend(context.Background(), block) require.Error(t, err) - require.True(t, errors.Is(err, realprotocol.ErrClusterNotFound), err) + require.ErrorIs(t, err, realprotocol.ErrClusterNotFound) require.True(t, st.IsInvalidExtensionError(err), err) }) } -func TestMakeValid(t *testing.T) { - t.Run("should trigger BlockProcessable with parent block", func(t *testing.T) { - consumer := &mockprotocol.Consumer{} - rootSnapshot := unittest.RootSnapshotFixture(participants) - head, err := rootSnapshot.Head() - require.NoError(t, err) - util.RunWithFullProtocolStateAndConsumer(t, rootSnapshot, consumer, func(db *badger.DB, state *protocol.MutableState) { - // create block2 and block3 - block2 := unittest.BlockWithParentFixture(head) - block2.SetPayload(flow.EmptyPayload()) - err := state.Extend(context.Background(), block2) - require.NoError(t, err) - - block3 := unittest.BlockWithParentFixture(block2.Header) - block3.SetPayload(flow.EmptyPayload()) - err = state.Extend(context.Background(), block3) - require.NoError(t, err) - - consumer.On("BlockProcessable", mock.Anything).Return() - - // make valid on block2 - err = state.MarkValid(block2.ID()) - require.NoError(t, err) - - // because the parent block is the root block, - // BlockProcessable is not triggered on root block. - consumer.AssertNotCalled(t, "BlockProcessable") - - err = state.MarkValid(block3.ID()) - require.NoError(t, err) - - // because the parent is not a root block, BlockProcessable event should be emitted - // block3's parent is block2 - consumer.AssertCalled(t, "BlockProcessable", block2.Header) - }) - }) -} - // If block B is finalized and contains a seal for block A, then A is the last sealed block func TestSealed(t *testing.T) { rootSnapshot := unittest.RootSnapshotFixture(participants) @@ -2002,11 +2162,11 @@ func TestHeaderInvalidTimestamp(t *testing.T) { // create a event consumer to test epoch transition events distributor := events.NewDistributor() - consumer := new(mockprotocol.Consumer) + consumer := mockprotocol.NewConsumer(t) distributor.AddConsumer(consumer) block, result, seal := unittest.BootstrapFixture(participants) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(block.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID())) rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) require.NoError(t, err) @@ -2030,11 +2190,10 @@ func TestHeaderInvalidTimestamp(t *testing.T) { }) } -func assertEpochEmergencyFallbackTriggered(t *testing.T, db *badger.DB) { - var triggered bool - err := db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) +func assertEpochEmergencyFallbackTriggered(t *testing.T, state realprotocol.State, expected bool) { + triggered, err := state.Params().EpochFallbackTriggered() require.NoError(t, err) - assert.True(t, triggered) + assert.Equal(t, expected, triggered) } // mockMetricsForRootSnapshot mocks the given metrics mock object to expect all @@ -2051,5 +2210,4 @@ func mockMetricsForRootSnapshot(metricsMock *mockmodule.ComplianceMetrics, rootS metricsMock.On("BlockFinalized", mock.Anything) metricsMock.On("FinalizedHeight", mock.Anything) metricsMock.On("SealedHeight", mock.Anything) - } diff --git a/state/protocol/badger/params.go b/state/protocol/badger/params.go index afd90fa6769..7f19d26234f 100644 --- a/state/protocol/badger/params.go +++ b/state/protocol/badger/params.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage/badger/operation" ) @@ -11,7 +12,9 @@ type Params struct { state *State } -func (p *Params) ChainID() (flow.ChainID, error) { +var _ protocol.Params = (*Params)(nil) + +func (p Params) ChainID() (flow.ChainID, error) { // retrieve root header root, err := p.Root() @@ -22,7 +25,7 @@ func (p *Params) ChainID() (flow.ChainID, error) { return root.ChainID, nil } -func (p *Params) SporkID() (flow.Identifier, error) { +func (p Params) SporkID() (flow.Identifier, error) { var sporkID flow.Identifier err := p.state.db.View(operation.RetrieveSporkID(&sporkID)) @@ -33,7 +36,17 @@ func (p *Params) SporkID() (flow.Identifier, error) { return sporkID, nil } -func (p *Params) ProtocolVersion() (uint, error) { +func (p Params) SporkRootBlockHeight() (uint64, error) { + var sporkRootBlockHeight uint64 + err := p.state.db.View(operation.RetrieveSporkRootBlockHeight(&sporkRootBlockHeight)) + if err != nil { + return 0, fmt.Errorf("could not get spork root block height: %w", err) + } + + return sporkRootBlockHeight, nil +} + +func (p Params) ProtocolVersion() (uint, error) { var version uint err := p.state.db.View(operation.RetrieveProtocolVersion(&version)) @@ -44,18 +57,30 @@ func (p *Params) ProtocolVersion() (uint, error) { return version, nil } -func (p *Params) Root() (*flow.Header, error) { +func (p Params) EpochCommitSafetyThreshold() (uint64, error) { - // retrieve the root height - var height uint64 - err := p.state.db.View(operation.RetrieveRootHeight(&height)) + var threshold uint64 + err := p.state.db.View(operation.RetrieveEpochCommitSafetyThreshold(&threshold)) if err != nil { - return nil, fmt.Errorf("could not retrieve root height: %w", err) + return 0, fmt.Errorf("could not get epoch commit safety threshold") } + return threshold, nil +} - // look up root header +func (p Params) EpochFallbackTriggered() (bool, error) { + var triggered bool + err := p.state.db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) + if err != nil { + return false, fmt.Errorf("could not check epoch fallback triggered: %w", err) + } + return triggered, nil +} + +func (p Params) Root() (*flow.Header, error) { + + // look up root block ID var rootID flow.Identifier - err = p.state.db.View(operation.LookupBlockHeight(height, &rootID)) + err := p.state.db.View(operation.LookupBlockHeight(p.state.rootHeight, &rootID)) if err != nil { return nil, fmt.Errorf("could not look up root header: %w", err) } @@ -69,18 +94,11 @@ func (p *Params) Root() (*flow.Header, error) { return header, nil } -func (p *Params) Seal() (*flow.Seal, error) { - - // retrieve the root height - var height uint64 - err := p.state.db.View(operation.RetrieveRootHeight(&height)) - if err != nil { - return nil, fmt.Errorf("could not retrieve root height: %w", err) - } +func (p Params) Seal() (*flow.Seal, error) { // look up root header var rootID flow.Identifier - err = p.state.db.View(operation.LookupBlockHeight(height, &rootID)) + err := p.state.db.View(operation.LookupBlockHeight(p.state.rootHeight, &rootID)) if err != nil { return nil, fmt.Errorf("could not look up root header: %w", err) } diff --git a/state/protocol/badger/snapshot.go b/state/protocol/badger/snapshot.go index 826421641f3..39193af0791 100644 --- a/state/protocol/badger/snapshot.go +++ b/state/protocol/badger/snapshot.go @@ -32,6 +32,8 @@ type Snapshot struct { blockID flow.Identifier // reference block for this snapshot } +var _ protocol.Snapshot = (*Snapshot)(nil) + func NewSnapshot(state *State, blockID flow.Identifier) *Snapshot { return &Snapshot{ state: state, @@ -47,7 +49,7 @@ func (s *Snapshot) Head() (*flow.Header, error) { // QuorumCertificate (QC) returns a valid quorum certificate pointing to the // header at this snapshot. With the exception of the root block, a valid child // block must be which contains the desired QC. The sentinel error -// state.NoValidChildBlockError is returned if the the QC is unknown. +// state.NoChildBlockError is returned if the the QC is unknown. // // For root block snapshots, returns the root quorum certificate. For all other // blocks, generates a quorum certificate from a valid child, if one exists. @@ -95,11 +97,13 @@ func (s *Snapshot) QuorumCertificate() (*flow.QuorumCertificate, error) { return qc, nil } -// validChild returns a child of the snapshot head that has been validated -// by HotStuff. Returns state.NoValidChildBlockError if no valid child exists. +// validChild returns a child of the snapshot head. Any valid child may be returned. +// Subsequent calls are not guaranteed to return the same child. +// Since blocks are fully validated before insertion to the state, all stored child +// blocks are valid and may be returned. // -// Any valid child may be returned. Subsequent calls are not guaranteed to -// return the same child. +// Error returns: +// - state.NoChildBlockError if no valid child exists. func (s *Snapshot) validChild() (*flow.Header, error) { var childIDs flow.IdentifierList @@ -108,30 +112,12 @@ func (s *Snapshot) validChild() (*flow.Header, error) { return nil, fmt.Errorf("could not look up children: %w", err) } - // find the first child that has been validated - validChildID := flow.ZeroID - for _, childID := range childIDs { - var valid bool - err = s.state.db.View(operation.RetrieveBlockValidity(childID, &valid)) - // skip blocks whose validity hasn't been checked yet - if errors.Is(err, storage.ErrNotFound) { - continue - } - if err != nil { - return nil, fmt.Errorf("failed to determine validity of child block %v: %w", childID, err) - } - if valid { - validChildID = childID - break - } - } - - if validChildID == flow.ZeroID { - return nil, state.NewNoValidChildBlockErrorf("block has no valid children (total children: %d)", len(childIDs)) + if len(childIDs) == 0 { + return nil, state.NewNoChildBlockErrorf("block (id=%x) has no children stored in the protocol state", s.blockID) } // get the header of the first child - child, err := s.state.headers.ByBlockID(validChildID) + child, err := s.state.headers.ByBlockID(childIDs[0]) return child, err } @@ -268,26 +254,53 @@ func (s *Snapshot) SealedResult() (*flow.ExecutionResult, *flow.Seal, error) { // SealingSegment will walk through the chain backward until we reach the block referenced // by the latest seal and build a SealingSegment. As we visit each block we check each execution -// receipt in the block's payload to make sure we have a corresponding execution result, any execution -// results missing from blocks are stored in the SealingSegment.ExecutionResults field. +// receipt in the block's payload to make sure we have a corresponding execution result, any +// execution results missing from blocks are stored in the `SealingSegment.ExecutionResults` field. +// See `model/flow/sealing_segment.md` for detailed technical specification of the Sealing Segment +// +// Expected errors during normal operations: +// - protocol.ErrSealingSegmentBelowRootBlock if sealing segment would stretch beyond the node's local history cut-off +// - protocol.UnfinalizedSealingSegmentError if sealing segment would contain unfinalized blocks (including orphaned blocks) func (s *Snapshot) SealingSegment() (*flow.SealingSegment, error) { - var rootHeight uint64 - err := s.state.db.View(operation.RetrieveRootHeight(&rootHeight)) - if err != nil { - return nil, fmt.Errorf("could not get root height: %w", err) + // Lets denote the highest block in the sealing segment `head` (initialized below). + // Based on the tech spec `flow/sealing_segment.md`, the Sealing Segment must contain contain + // enough history to satisfy _all_ of the following conditions: + // (i) The highest sealed block as of `head` needs to be included in the sealing segment. + // This is relevant if `head` does not contain any seals. + // (ii) All blocks that are sealed by `head`. This is relevant if head` contains _multiple_ seals. + // (iii) The sealing segment should contain the history back to (including): + // limitHeight := max(head.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + // Per convention, we include the blocks for (i) in the `SealingSegment.Blocks`, while the + // additional blocks for (ii) and optionally (iii) are contained in as `SealingSegment.ExtraBlocks`. + head, err := s.state.blocks.ByID(s.blockID) + if err != nil { + return nil, fmt.Errorf("could not get snapshot's reference block: %w", err) + } + if head.Header.Height < s.state.rootHeight { + return nil, protocol.ErrSealingSegmentBelowRootBlock } - head, err := s.Head() + + // Verify that head of sealing segment is finalized. + finalizedBlockAtHeight, err := s.state.headers.BlockIDByHeight(head.Header.Height) if err != nil { - return nil, fmt.Errorf("could not get snapshot reference block: %w", err) + if errors.Is(err, storage.ErrNotFound) { + return nil, protocol.NewUnfinalizedSealingSegmentErrorf("head of sealing segment at height %d is not finalized: %w", head.Header.Height, err) + } + return nil, fmt.Errorf("exception while retrieving finzalized bloc, by height: %w", err) } - if head.Height < rootHeight { - return nil, protocol.ErrSealingSegmentBelowRootBlock + if finalizedBlockAtHeight != s.blockID { // comparison of fixed-length arrays + return nil, protocol.NewUnfinalizedSealingSegmentErrorf("head of sealing segment is orphaned, finalized block at height %d is %x", head.Header.Height, finalizedBlockAtHeight) } + // STEP (i): highest sealed block as of `head` must be included. seal, err := s.state.seals.HighestInFork(s.blockID) if err != nil { return nil, fmt.Errorf("could not get seal for sealing segment: %w", err) } + blockSealedAtHead, err := s.state.headers.ByBlockID(seal.BlockID) + if err != nil { + return nil, fmt.Errorf("could not get block: %w", err) + } // walk through the chain backward until we reach the block referenced by // the latest seal - the returned segment includes this block @@ -306,12 +319,60 @@ func (s *Snapshot) SealingSegment() (*flow.SealingSegment, error) { return nil } - err = fork.TraverseForward(s.state.headers, s.blockID, scraper, fork.IncludingBlock(seal.BlockID)) if err != nil { return nil, fmt.Errorf("could not traverse sealing segment: %w", err) } + // STEP (ii): extend history down to the lowest block, whose seal is included in `head` + lowestSealedByHead := blockSealedAtHead + for _, sealInHead := range head.Payload.Seals { + h, e := s.state.headers.ByBlockID(sealInHead.BlockID) + if e != nil { + return nil, fmt.Errorf("could not get block (id=%x) for seal: %w", seal.BlockID, e) // storage.ErrNotFound or exception + } + if h.Height < lowestSealedByHead.Height { + lowestSealedByHead = h + } + } + + // STEP (iii): extended history to allow checking for duplicated collections, i.e. + // limitHeight = max(head.Height - flow.DefaultTransactionExpiry, SporkRootBlockHeight) + limitHeight := s.state.sporkRootBlockHeight + if head.Header.Height > s.state.sporkRootBlockHeight+flow.DefaultTransactionExpiry { + limitHeight = head.Header.Height - flow.DefaultTransactionExpiry + } + + // As we have to satisfy (ii) _and_ (iii), we have to take the longest history, i.e. the lowest height. + if lowestSealedByHead.Height < limitHeight { + limitHeight = lowestSealedByHead.Height + if limitHeight < s.state.sporkRootBlockHeight { // sanity check; should never happen + return nil, fmt.Errorf("unexpected internal error: calculated history-cutoff at height %d, which is lower than the spork's root height %d", limitHeight, s.state.sporkRootBlockHeight) + } + } + if limitHeight < blockSealedAtHead.Height { + // we need to include extra blocks in sealing segment + extraBlocksScraper := func(header *flow.Header) error { + blockID := header.ID() + block, err := s.state.blocks.ByID(blockID) + if err != nil { + return fmt.Errorf("could not get block: %w", err) + } + + err = builder.AddExtraBlock(block) + if err != nil { + return fmt.Errorf("could not add block to sealing segment: %w", err) + } + + return nil + } + + err = fork.TraverseBackward(s.state.headers, blockSealedAtHead.ParentID, extraBlocksScraper, fork.IncludingHeight(limitHeight)) + if err != nil { + return nil, fmt.Errorf("could not traverse extra blocks for sealing segment: %w", err) + } + } + segment, err := builder.SealingSegment() if err != nil { return nil, fmt.Errorf("could not build sealing segment: %w", err) @@ -328,22 +389,6 @@ func (s *Snapshot) Descendants() ([]flow.Identifier, error) { return descendants, nil } -func (s *Snapshot) ValidDescendants() ([]flow.Identifier, error) { - valid, err := s.lookupValidity(s.blockID) - if err != nil { - return nil, fmt.Errorf("could not determine validity of block %v: %w", s.blockID, err) - } - if !valid { - return []flow.Identifier{}, nil - } - - descendants, err := s.validDescendants(s.blockID) - if err != nil { - return nil, fmt.Errorf("failed to traverse the descendants tree of block %v: %w", s.blockID, err) - } - return descendants, nil -} - func (s *Snapshot) lookupChildren(blockID flow.Identifier) ([]flow.Identifier, error) { var children flow.IdentifierList err := s.state.db.View(procedure.LookupBlockChildren(blockID, &children)) @@ -353,45 +398,6 @@ func (s *Snapshot) lookupChildren(blockID flow.Identifier) ([]flow.Identifier, e return children, nil } -func (s *Snapshot) lookupValidity(blockID flow.Identifier) (bool, error) { - valid := false - err := s.state.db.View(operation.RetrieveBlockValidity(blockID, &valid)) - if err != nil { - // We only store the validity flag for blocks that have been marked valid. - // For blocks that haven't been marked valid (yet), the flag is simply absent. - if !errors.Is(err, storage.ErrNotFound) { - return false, fmt.Errorf("could not retrieve validity of block %v: %w", blockID, err) - } - } - return valid, nil -} - -func (s *Snapshot) validDescendants(blockID flow.Identifier) ([]flow.Identifier, error) { - var descendantIDs []flow.Identifier - - children, err := s.lookupChildren(blockID) - if err != nil { - return nil, err - } - - for _, descendantID := range children { - valid, err := s.lookupValidity(descendantID) - if err != nil { - return nil, err - } - - if valid { - descendantIDs = append(descendantIDs, descendantID) - additionalIDs, err := s.validDescendants(descendantID) - if err != nil { - return nil, err - } - descendantIDs = append(descendantIDs, additionalIDs...) - } - } - return descendantIDs, nil -} - func (s *Snapshot) descendants(blockID flow.Identifier) ([]flow.Identifier, error) { descendantIDs, err := s.lookupChildren(blockID) if err != nil { @@ -410,7 +416,7 @@ func (s *Snapshot) descendants(blockID flow.Identifier) ([]flow.Identifier, erro // RandomSource returns the seed for the current block snapshot. // Expected error returns: -// * state.NoValidChildBlockError if no valid child is known +// * state.NoChildBlockError if no valid child is known func (s *Snapshot) RandomSource() ([]byte, error) { // CASE 1: for the root block, generate the seed from the root qc diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index b404b0bada2..dd0d24f9e7f 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -16,6 +16,8 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/module/signature" + statepkg "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/state/protocol" bprotocol "github.com/onflow/flow-go/state/protocol/badger" "github.com/onflow/flow-go/state/protocol/inmem" @@ -77,10 +79,7 @@ func TestSnapshot_Params(t *testing.T) { const nBlocks = 10 for i := 0; i < nBlocks; i++ { next := unittest.BlockWithParentFixture(head) - err = state.Extend(context.Background(), next) - require.NoError(t, err) - err = state.Finalize(context.Background(), next.ID()) - require.NoError(t, err) + buildFinalizedBlock(t, state, next) head = next.Header } @@ -137,39 +136,6 @@ func TestSnapshot_Descendants(t *testing.T) { }) } -// TestSnapshot_ValidDescendants builds a sample chain with next structure: -// -// A (finalized) <- B <- C <- D <- E <- F -// <- G <- H <- I <- J -// -// snapshot.Descendants has to return [B, C, D, E, G, H, I]. [F, J] should be excluded because they aren't valid -func TestSnapshot_ValidDescendants(t *testing.T) { - participants := unittest.IdentityListFixture(5, unittest.WithAllRoles()) - rootSnapshot := unittest.RootSnapshotFixture(participants) - head, err := rootSnapshot.Head() - require.NoError(t, err) - util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.MutableState) { - var expectedBlocks []flow.Identifier - for i := 5; i > 3; i-- { - fork := unittest.ChainFixtureFrom(i, head) - for blockIndex, block := range fork { - err := state.Extend(context.Background(), block) - require.NoError(t, err) - // skip last block from fork - if blockIndex < len(fork)-1 { - err = state.MarkValid(block.ID()) - require.NoError(t, err) - expectedBlocks = append(expectedBlocks, block.ID()) - } - } - } - - pendingBlocks, err := state.AtBlockID(head.ID()).ValidDescendants() - require.NoError(t, err) - require.ElementsMatch(t, expectedBlocks, pendingBlocks) - }) -} - func TestIdentities(t *testing.T) { identities := unittest.IdentityListFixture(5, unittest.WithAllRoles()) rootSnapshot := unittest.RootSnapshotFixture(identities) @@ -213,7 +179,7 @@ func TestClusters(t *testing.T) { identities := append(unittest.IdentityListFixture(4, unittest.WithAllRolesExcept(flow.RoleCollection)), collectors...) root, result, seal := unittest.BootstrapFixture(identities) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(root.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) commit := result.ServiceEvents[1].Event.(*flow.EpochCommit) setup.Assignments = unittest.ClusterAssignment(uint(nClusters), collectors) @@ -261,6 +227,7 @@ func TestSealingSegment(t *testing.T) { assert.Len(t, actual.ExecutionResults, 1) assert.Len(t, actual.Blocks, 1) + assert.Empty(t, actual.ExtraBlocks) unittest.AssertEqualBlocksLenAndOrder(t, expected.Blocks, actual.Blocks) assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(head.ID())) @@ -270,12 +237,12 @@ func TestSealingSegment(t *testing.T) { // test sealing segment for non-root segment where the latest seal is the // root seal, but the segment contains more than the root block. // ROOT <- B1 - // Expected sealing segment: [ROOT, B1] + // Expected sealing segment: [ROOT, B1], extra blocks: [] t.Run("non-root with root seal as latest seal", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build an extra block on top of root block1 := unittest.BlockWithParentFixture(head) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) segment, err := state.AtBlockID(block1.ID()).SealingSegment() require.NoError(t, err) @@ -285,8 +252,9 @@ func TestSealingSegment(t *testing.T) { // sealing segment should contain B1 and B2 // B2 is reference of snapshot, B1 is latest sealed - unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{rootSnapshot.Encodable().SealingSegment.Lowest(), block1}, segment.Blocks) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{rootSnapshot.Encodable().SealingSegment.Sealed(), block1}, segment.Blocks) assert.Len(t, segment.ExecutionResults, 1) + assert.Empty(t, segment.ExtraBlocks) assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block1.ID())) }) }) @@ -294,22 +262,25 @@ func TestSealingSegment(t *testing.T) { // test sealing segment for non-root segment with simple sealing structure // (no blocks in between reference block and latest sealed) // ROOT <- B1 <- B2(S1) - // Expected sealing segment: [B1, B2] + // Expected sealing segment: [B1, B2], extra blocks: [ROOT] t.Run("non-root", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) // build a block sealing block1 block2 := unittest.BlockWithParentFixture(block1.Header) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1), unittest.WithSeals(seal1))) - buildBlock(t, state, block2) + buildFinalizedBlock(t, state, block2) segment, err := state.AtBlockID(block2.ID()).SealingSegment() require.NoError(t, err) + require.Len(t, segment.ExtraBlocks, 1) + assert.Equal(t, segment.ExtraBlocks[0].Header.Height, head.Height) + // build a valid child B3 to ensure we have a QC buildBlock(t, state, unittest.BlockWithParentFixture(block2.Header)) @@ -324,19 +295,19 @@ func TestSealingSegment(t *testing.T) { // test sealing segment for sealing segment with a large number of blocks // between the reference block and latest sealed // ROOT <- B1 <- .... <- BN(S1) - // Expected sealing segment: [B1, ..., BN] + // Expected sealing segment: [B1, ..., BN], extra blocks: [ROOT] t.Run("long sealing segment", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) parent := block1 // build a large chain of intermediary blocks for i := 0; i < 100; i++ { next := unittest.BlockWithParentFixture(parent.Header) - buildBlock(t, state, next) + buildFinalizedBlock(t, state, next) parent = next } @@ -344,10 +315,10 @@ func TestSealingSegment(t *testing.T) { blockN := unittest.BlockWithParentFixture(parent.Header) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) blockN.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1), unittest.WithSeals(seal1))) - buildBlock(t, state, blockN) + buildFinalizedBlock(t, state, blockN) // build a valid child B3 to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(blockN.Header)) + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(blockN.Header)) segment, err := state.AtBlockID(blockN.ID()).SealingSegment() require.NoError(t, err) @@ -355,6 +326,8 @@ func TestSealingSegment(t *testing.T) { assert.Len(t, segment.ExecutionResults, 1) // sealing segment should cover range [B1, BN] assert.Len(t, segment.Blocks, 102) + assert.Len(t, segment.ExtraBlocks, 1) + assert.Equal(t, segment.ExtraBlocks[0].Header.Height, head.Height) // first and last blocks should be B1, BN assert.Equal(t, block1.ID(), segment.Blocks[0].ID()) assert.Equal(t, blockN.ID(), segment.Blocks[101].ID()) @@ -365,33 +338,33 @@ func TestSealingSegment(t *testing.T) { // test sealing segment where the segment blocks contain seals for // ancestor blocks prior to the sealing segment // ROOT <- B1 <- B2(R1) <- B3 <- B4(R2, S1) <- B5 <- B6(S2) - // Expected sealing segment: [B2, B3, B4] + // Expected sealing segment: [B2, B3, B4], Extra blocks: [ROOT, B1] t.Run("overlapping sealing segment", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { block1 := unittest.BlockWithParentFixture(head) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) - buildBlock(t, state, block2) + buildFinalizedBlock(t, state, block2) receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) block3 := unittest.BlockWithParentFixture(block2.Header) - buildBlock(t, state, block3) + buildFinalizedBlock(t, state, block3) block4 := unittest.BlockWithParentFixture(block3.Header) block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) - buildBlock(t, state, block4) + buildFinalizedBlock(t, state, block4) block5 := unittest.BlockWithParentFixture(block4.Header) - buildBlock(t, state, block5) + buildFinalizedBlock(t, state, block5) block6 := unittest.BlockWithParentFixture(block5.Header) block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) - buildBlock(t, state, block6) + buildFinalizedBlock(t, state, block6) segment, err := state.AtBlockID(block6.ID()).SealingSegment() require.NoError(t, err) @@ -402,6 +375,7 @@ func TestSealingSegment(t *testing.T) { // sealing segment should be [B2, B3, B4, B5, B6] require.Len(t, segment.Blocks, 5) unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5, block6}, segment.Blocks) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1}, segment.ExtraBlocks[1:]) require.Len(t, segment.ExecutionResults, 1) assertSealingSegmentBlocksQueryableAfterBootstrap(t, state.AtBlockID(block6.ID())) @@ -439,11 +413,11 @@ func TestSealingSegment(t *testing.T) { block5 := unittest.BlockWithParentFixture(block4.Header) block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(sealC))) - buildBlock(t, state, block1) - buildBlock(t, state, block2) - buildBlock(t, state, block3) - buildBlock(t, state, block4) - buildBlock(t, state, block5) + buildFinalizedBlock(t, state, block1) + buildFinalizedBlock(t, state, block2) + buildFinalizedBlock(t, state, block3) + buildFinalizedBlock(t, state, block4) + buildFinalizedBlock(t, state, block5) segment, err := state.AtBlockID(block5.ID()).SealingSegment() require.NoError(t, err) @@ -497,11 +471,11 @@ func TestSealingSegment(t *testing.T) { block5 := unittest.BlockWithParentFixture(block4.Header) block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal))) - buildBlock(t, state, block1) - buildBlock(t, state, block2) - buildBlock(t, state, block3) - buildBlock(t, state, block4) - buildBlock(t, state, block5) + buildFinalizedBlock(t, state, block1) + buildFinalizedBlock(t, state, block2) + buildFinalizedBlock(t, state, block3) + buildFinalizedBlock(t, state, block4) + buildFinalizedBlock(t, state, block5) segment, err := state.AtBlockID(block5.ID()).SealingSegment() require.NoError(t, err) @@ -522,33 +496,33 @@ func TestSealingSegment(t *testing.T) { // Test the case where the reference block of the snapshot contains no seal. // We should consider the latest seal in a prior block. // ROOT <- B1 <- B2(R1) <- B3 <- B4(S1) <- B5 - // Expected sealing segment: [B1, B2, B3, B4, B5] + // Expected sealing segment: [B1, B2, B3, B4, B5], Extra blocks: [ROOT] t.Run("sealing segment where highest block in segment does not seal lowest", func(t *testing.T) { util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { // build a block to seal block1 := unittest.BlockWithParentFixture(head) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) // build a block sealing block1 block2 := unittest.BlockWithParentFixture(block1.Header) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) - buildBlock(t, state, block2) + buildFinalizedBlock(t, state, block2) block3 := unittest.BlockWithParentFixture(block2.Header) - buildBlock(t, state, block3) + buildFinalizedBlock(t, state, block3) block4 := unittest.BlockWithParentFixture(block3.Header) block4.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1))) - buildBlock(t, state, block4) + buildFinalizedBlock(t, state, block4) block5 := unittest.BlockWithParentFixture(block4.Header) - buildBlock(t, state, block5) + buildFinalizedBlock(t, state, block5) snapshot := state.AtBlockID(block5.ID()) // build a valid child to ensure we have a QC - buildBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(block5.Header)) segment, err := snapshot.SealingSegment() require.NoError(t, err) @@ -560,6 +534,230 @@ func TestSealingSegment(t *testing.T) { assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) }) }) + // Test the case where the reference block of the snapshot contains seals for blocks that are lower than the lowest sealing segment's block. + // This test case specifically checks if sealing segment includes both highest and lowest block sealed by head. + // ROOT <- B1 <- B2 <- B3(Seal_B1) <- B4 <- ... <- LastBlock(Seal_B2, Seal_B3, Seal_B4) + // Expected sealing segment: [B4, ..., B5], Extra blocks: [B2, B3] + t.Run("highest block seals outside segment", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // build a block to seal + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + + // build a block sealing block1 + block2 := unittest.BlockWithParentFixture(block1.Header) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + block3 := unittest.BlockWithParentFixture(block2.Header) + block3.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1), unittest.WithReceipts(receipt2))) + buildFinalizedBlock(t, state, block3) + + receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3) + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3))) + buildFinalizedBlock(t, state, block4) + + // build chain, so it's long enough to not target blocks as inside of flow.DefaultTransactionExpiry window. + parent := block4 + for i := 0; i < 1.5*flow.DefaultTransactionExpiry; i++ { + next := unittest.BlockWithParentFixture(parent.Header) + next.Header.View = next.Header.Height + 1 // set view so we are still in the same epoch + buildFinalizedBlock(t, state, next) + parent = next + } + + receipt4, seal4 := unittest.ReceiptAndSealForBlock(block4) + lastBlock := unittest.BlockWithParentFixture(parent.Header) + lastBlock.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2, seal3, seal4), unittest.WithReceipts(receipt4))) + buildFinalizedBlock(t, state, lastBlock) + + snapshot := state.AtBlockID(lastBlock.ID()) + + // build a valid child to ensure we have a QC + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(lastBlock.Header)) + + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + assert.Equal(t, lastBlock.Header, segment.Highest().Header) + assert.Equal(t, block4.Header, segment.Sealed().Header) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3}, segment.ExtraBlocks) + assert.Len(t, segment.ExecutionResults, 2) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + }) + }) +} + +// TestSealingSegment_FailureCases verifies that SealingSegment construction fails with expected sentinel +// errors in case the caller violates the API contract: +// 1. The lowest block that can serve as head of a SealingSegment is the node's local root block. +// 2. Unfinalized blocks cannot serve as head of a SealingSegment. There are two distinct sub-cases: +// (2a) A pending block is chosen as head; at this height no block has been finalized. +// (2b) An orphaned block is chosen as head; at this height a block other than the orphaned has been finalized. +func TestSealingSegment_FailureCases(t *testing.T) { + sporkRootSnapshot := unittest.RootSnapshotFixture(unittest.CompleteIdentitySet()) + sporkRoot, err := sporkRootSnapshot.Head() + require.NoError(t, err) + + // SCENARIO 1. + // Here, we want to specifically test correct handling of the edge case, where a block exists in storage + // that has _lower height_ than the node's local root block. Such blocks are typically contained in the + // bootstrapping data, such that all entities referenced in the local root block can be resolved. + // Is is possible to retrieve blocks that are lower than the local root block from storage, directly + // via their ID. Despite these blocks existing in storage, SealingSegment construction should be + // because the known history is potentially insufficient when going below the root block. + t.Run("sealing segment from block below local state root", func(t *testing.T) { + // Step I: constructing bootstrapping snapshot with some short history: + // + // ╭───── finalized blocks ─────╮ + // <- b1 <- b2 <- b3(seal(b1)) <- + // └── head ──┘ + // + b1 := unittest.BlockWithParentFixture(sporkRoot) // construct block b1, append to state and finalize + b2 := unittest.BlockWithParentFixture(b1.Header) // construct block b2, append to state and finalize + b3 := unittest.BlockWithParentFixture(b2.Header) // construct block b3 with seal for b1, append it to state and finalize + receipt, seal := unittest.ReceiptAndSealForBlock(b1) + b3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt), unittest.WithSeals(seal))) + + multipleBlockSnapshot := snapshotAfter(t, sporkRootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { + for _, b := range []*flow.Block{b1, b2, b3} { + buildFinalizedBlock(t, state, b) + } + require.NoError(t, state.Extend(context.Background(), unittest.BlockWithParentFixture(b3.Header))) // add child of b3 to ensure we have a QC for b3 + return state.AtBlockID(b3.ID()) + }) + + // Step 2: bootstrapping new state based on sealing segment whose head is block b3. + // Thereby, the state should have b3 as its local root block. In addition, the blocks contained in the sealing + // segment, such as b2 should be stored in the state. + util.RunWithFollowerProtocolState(t, multipleBlockSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + localStateRootBlock, err := state.Params().Root() + require.NoError(t, err) + assert.Equal(t, b3.ID(), localStateRootBlock.ID()) + + // verify that b2 is known to the protocol state, but constructing a sealing segment fails + _, err = state.AtBlockID(b2.ID()).Head() + require.NoError(t, err) + _, err = state.AtBlockID(b2.ID()).SealingSegment() + assert.ErrorIs(t, err, protocol.ErrSealingSegmentBelowRootBlock) + + // lowest block that allows for sealing segment construction is root block: + _, err = state.AtBlockID(b3.ID()).SealingSegment() + require.NoError(t, err) + }) + }) + + // SCENARIO 2a: A pending block is chosen as head; at this height no block has been finalized. + t.Run("sealing segment from unfinalized, pending block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + // add _unfinalized_ blocks b1 and b2 to state (block b5 is necessary, so b1 has a QC, which is a consistency requirement for subsequent finality) + b1 := unittest.BlockWithParentFixture(sporkRoot) + require.NoError(t, state.Extend(context.Background(), b1)) + require.NoError(t, state.Extend(context.Background(), unittest.BlockWithParentFixture(b1.Header))) // adding block b5 (providing required QC for b1) + + // consistency check: there should be no finalized block in the protocol state at height `b1.Height` + _, err := state.AtHeight(b1.Header.Height).Head() // expect statepkg.ErrUnknownSnapshotReference as only finalized blocks are indexed by height + assert.ErrorIs(t, err, statepkg.ErrUnknownSnapshotReference) + + // requesting a sealing segment from block b1 should fail, as b1 is not yet finalized + _, err = state.AtBlockID(b1.ID()).SealingSegment() + assert.True(t, protocol.IsUnfinalizedSealingSegmentError(err)) + }) + }) + + // SCENARIO 2b: An orphaned block is chosen as head; at this height a block other than the orphaned has been finalized. + t.Run("sealing segment from orphaned block", func(t *testing.T) { + util.RunWithFollowerProtocolState(t, sporkRootSnapshot, func(db *badger.DB, state *bprotocol.FollowerState) { + orphaned := unittest.BlockWithParentFixture(sporkRoot) + require.NoError(t, state.Extend(context.Background(), orphaned)) + require.NoError(t, state.Extend(context.Background(), unittest.BlockWithParentFixture(orphaned.Header))) + buildFinalizedBlock(t, state, unittest.BlockWithParentFixture(sporkRoot)) + + // consistency check: the finalized block at height `orphaned.Height` should be different than `orphaned` + h, err := state.AtHeight(orphaned.Header.Height).Head() + require.NoError(t, err) + require.NotEqual(t, h.ID(), orphaned.ID()) + + // requesting a sealing segment from orphaned block should fail, as it is not finalized + _, err = state.AtBlockID(orphaned.ID()).SealingSegment() + assert.True(t, protocol.IsUnfinalizedSealingSegmentError(err)) + }) + }) + +} + +// TestBootstrapSealingSegmentWithExtraBlocks test sealing segment where the segment blocks contain collection +// guarantees referencing blocks prior to the sealing segment. After bootstrapping from sealing segment we should be able to +// extend with B7 with contains a guarantee referring B1. +// ROOT <- B1 <- B2(R1) <- B3 <- B4(S1) <- B5 <- B6(S2) +// Expected sealing segment: [B2, B3, B4, B5, B6], Extra blocks: [ROOT, B1] +func TestBootstrapSealingSegmentWithExtraBlocks(t *testing.T) { + identities := unittest.CompleteIdentitySet() + rootSnapshot := unittest.RootSnapshotFixture(identities) + rootEpoch := rootSnapshot.Epochs().Current() + cluster, err := rootEpoch.Cluster(0) + require.NoError(t, err) + collID := cluster.Members()[0].NodeID + head, err := rootSnapshot.Head() + require.NoError(t, err) + util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *bprotocol.MutableState) { + block1 := unittest.BlockWithParentFixture(head) + buildFinalizedBlock(t, state, block1) + receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) + + block2 := unittest.BlockWithParentFixture(block1.Header) + block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1))) + buildFinalizedBlock(t, state, block2) + + receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2) + + block3 := unittest.BlockWithParentFixture(block2.Header) + buildFinalizedBlock(t, state, block3) + + block4 := unittest.BlockWithParentFixture(block3.Header) + block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2), unittest.WithSeals(seal1))) + buildFinalizedBlock(t, state, block4) + + block5 := unittest.BlockWithParentFixture(block4.Header) + buildFinalizedBlock(t, state, block5) + + block6 := unittest.BlockWithParentFixture(block5.Header) + block6.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal2))) + buildFinalizedBlock(t, state, block6) + + snapshot := state.AtBlockID(block6.ID()) + segment, err := snapshot.SealingSegment() + require.NoError(t, err) + + // build a valid child to ensure we have a QC + buildBlock(t, state, unittest.BlockWithParentFixture(block6.Header)) + + // sealing segment should be [B2, B3, B4, B5, B6] + require.Len(t, segment.Blocks, 5) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block2, block3, block4, block5, block6}, segment.Blocks) + unittest.AssertEqualBlocksLenAndOrder(t, []*flow.Block{block1}, segment.ExtraBlocks[1:]) + require.Len(t, segment.ExecutionResults, 1) + + assertSealingSegmentBlocksQueryableAfterBootstrap(t, snapshot) + + // bootstrap from snapshot + util.RunWithFullProtocolState(t, snapshot, func(db *badger.DB, state *bprotocol.MutableState) { + block7 := unittest.BlockWithParentFixture(block6.Header) + guarantee := unittest.CollectionGuaranteeFixture(unittest.WithCollRef(block1.ID())) + guarantee.ChainID = cluster.ChainID() + + signerIndices, err := signature.EncodeSignersToIndices( + []flow.Identifier{collID}, []flow.Identifier{collID}) + require.NoError(t, err) + guarantee.SignerIndices = signerIndices + + block7.SetPayload(unittest.PayloadFixture(unittest.WithGuarantees(guarantee))) + buildBlock(t, state, block7) + }) + }) } func TestLatestSealedResult(t *testing.T) { @@ -708,16 +906,12 @@ func TestQuorumCertificate(t *testing.T) { block1.SetPayload(flow.EmptyPayload()) err := state.Extend(context.Background(), block1) require.Nil(t, err) - err = state.MarkValid(block1.ID()) - require.Nil(t, err) // add a valid child to block1 block2 := unittest.BlockWithParentFixture(block1.Header) block2.SetPayload(flow.EmptyPayload()) err = state.Extend(context.Background(), block2) require.Nil(t, err) - err = state.MarkValid(block2.ID()) - require.Nil(t, err) // should be able to get QC/seed qc, err := state.AtBlockID(block1.ID()).QuorumCertificate() @@ -1041,7 +1235,7 @@ func TestSnapshot_PostSporkIdentities(t *testing.T) { root, result, seal := unittest.BootstrapFixture(expected, func(block *flow.Block) { block.Header.ParentID = unittest.IdentifierFixture() }) - qc := unittest.QuorumCertificateFixture(unittest.QCWithBlockID(root.ID())) + qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(root.ID())) rootSnapshot, err := inmem.SnapshotFromBootstrapState(root, result, seal, qc) require.NoError(t, err) diff --git a/state/protocol/badger/state.go b/state/protocol/badger/state.go index f3c6a4da2a3..c867c607559 100644 --- a/state/protocol/badger/state.go +++ b/state/protocol/badger/state.go @@ -8,6 +8,7 @@ import ( "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" statepkg "github.com/onflow/flow-go/state" @@ -30,6 +31,10 @@ type State struct { commits storage.EpochCommits statuses storage.EpochStatuses } + // cache the root height because it cannot change over the lifecycle of a protocol state instance + rootHeight uint64 + // cache the spork root block height because it cannot change over the lifecycle of a protocol state instance + sporkRootBlockHeight uint64 } type BootstrapConfig struct { @@ -93,7 +98,7 @@ func Bootstrap( // oldest ancestor and head is the newest child in the segment // TAIL <- ... <- HEAD highest := segment.Highest() // reference block of the snapshot - lowest := segment.Lowest() // last sealed block + lowest := segment.Sealed() // last sealed block // 1) bootstrap the sealing segment err = state.bootstrapSealingSegment(segment, highest)(tx) @@ -118,7 +123,7 @@ func Bootstrap( } // 4) initialize values related to the epoch logic - err = state.bootstrapEpoch(root, !config.SkipNetworkAddressValidation)(tx) + err = state.bootstrapEpoch(root.Epochs(), segment, !config.SkipNetworkAddressValidation)(tx) if err != nil { return fmt.Errorf("could not bootstrap epoch values: %w", err) } @@ -147,6 +152,12 @@ func Bootstrap( return nil, fmt.Errorf("bootstrapping failed: %w", err) } + // populate the protocol state cache + err = state.populateCache() + if err != nil { + return nil, fmt.Errorf("failed to populate cache: %w", err) + } + return state, nil } @@ -174,17 +185,26 @@ func (state *State) bootstrapSealingSegment(segment *flow.SealingSegment, head * } } - for i, block := range segment.Blocks { + for _, block := range segment.ExtraBlocks { blockID := block.ID() height := block.Header.Height - err := state.blocks.StoreTx(block)(tx) if err != nil { return fmt.Errorf("could not insert root block: %w", err) } - err = transaction.WithTx(operation.InsertBlockValidity(blockID, true))(tx) + err = transaction.WithTx(operation.IndexBlockHeight(height, blockID))(tx) + if err != nil { + return fmt.Errorf("could not index root block segment (id=%x): %w", blockID, err) + } + } + + for i, block := range segment.Blocks { + blockID := block.ID() + height := block.Header.Height + + err := state.blocks.StoreTx(block)(tx) if err != nil { - return fmt.Errorf("could not mark root block as valid: %w", err) + return fmt.Errorf("could not insert root block: %w", err) } err = transaction.WithTx(operation.IndexBlockHeight(height, blockID))(tx) if err != nil { @@ -235,21 +255,52 @@ func (state *State) bootstrapStatePointers(root protocol.Snapshot) func(*badger. return fmt.Errorf("could not get sealing segment: %w", err) } highest := segment.Highest() - lowest := segment.Lowest() + lowest := segment.Sealed() // find the finalized seal that seals the lowest block, meaning seal.BlockID == lowest.ID() seal, err := segment.FinalizedSeal() if err != nil { return fmt.Errorf("could not get finalized seal from sealing segment: %w", err) } + safetyData := &hotstuff.SafetyData{ + LockedOneChainView: highest.Header.View, + HighestAcknowledgedView: highest.Header.View, + } + + // Per convention, all blocks in the sealing segment must be finalized. Therefore, a QC must + // exist for the `highest` block in the sealing segment. The QC for `highest` should be + // contained in the `root` Snapshot and returned by `root.QuorumCertificate()`. Otherwise, + // the Snapshot is incomplete, because consensus nodes require this QC. To reduce the chance of + // accidental misconfiguration undermining consensus liveness, we do the following sanity checks: + // * `rootQC` should not be nil + // * `rootQC` should be for `highest` block, i.e. its view and blockID should match + rootQC, err := root.QuorumCertificate() + if err != nil { + return fmt.Errorf("could not get root QC: %w", err) + } + if rootQC == nil { + return fmt.Errorf("QC for highest (finalized) block in sealing segment cannot be nil") + } + if rootQC.View != highest.Header.View { + return fmt.Errorf("root QC's view %d does not match the highest block in sealing segment (view %d)", rootQC.View, highest.Header.View) + } + if rootQC.BlockID != highest.Header.ID() { + return fmt.Errorf("root QC is for block %v, which does not match the highest block %v in sealing segment", rootQC.BlockID, highest.Header.ID()) + } + + livenessData := &hotstuff.LivenessData{ + CurrentView: highest.Header.View + 1, + NewestQC: rootQC, + } + // insert initial views for HotStuff - err = operation.InsertStartedView(highest.Header.ChainID, highest.Header.View)(tx) + err = operation.InsertSafetyData(highest.Header.ChainID, safetyData)(tx) if err != nil { - return fmt.Errorf("could not insert started view: %w", err) + return fmt.Errorf("could not insert safety data: %w", err) } - err = operation.InsertVotedView(highest.Header.ChainID, highest.Header.View)(tx) + err = operation.InsertLivenessData(highest.Header.ChainID, livenessData)(tx) if err != nil { - return fmt.Errorf("could not insert started view: %w", err) + return fmt.Errorf("could not insert liveness data: %w", err) } // insert height pointers @@ -279,11 +330,11 @@ func (state *State) bootstrapStatePointers(root protocol.Snapshot) func(*badger. // // The root snapshot's sealing segment must not straddle any epoch transitions // or epoch phase transitions. -func (state *State) bootstrapEpoch(root protocol.Snapshot, verifyNetworkAddress bool) func(*transaction.Tx) error { +func (state *State) bootstrapEpoch(epochs protocol.EpochQuery, segment *flow.SealingSegment, verifyNetworkAddress bool) func(*transaction.Tx) error { return func(tx *transaction.Tx) error { - previous := root.Epochs().Previous() - current := root.Epochs().Current() - next := root.Epochs().Next() + previous := epochs.Previous() + current := epochs.Current() + next := epochs.Next() // build the status as we go status := new(flow.EpochStatus) @@ -358,7 +409,7 @@ func (state *State) bootstrapEpoch(root protocol.Snapshot, verifyNetworkAddress setups = append(setups, setup) status.NextEpoch.SetupID = setup.ID() commit, err := protocol.ToEpochCommit(next) - if err != nil && !errors.Is(err, protocol.ErrEpochNotCommitted) { + if err != nil && !errors.Is(err, protocol.ErrNextEpochNotCommitted) { return fmt.Errorf("could not get next epoch commit event: %w", err) } if err == nil { @@ -394,11 +445,7 @@ func (state *State) bootstrapEpoch(root protocol.Snapshot, verifyNetworkAddress // NOTE: as specified in the godoc, this code assumes that each block // in the sealing segment in within the same phase within the same epoch. - segment, err := root.SealingSegment() - if err != nil { - return fmt.Errorf("could not get sealing segment: %w", err) - } - for _, block := range segment.Blocks { + for _, block := range segment.AllBlocks() { blockID := block.ID() err = state.epoch.statuses.StoreTx(blockID, status)(tx) if err != nil { @@ -425,6 +472,15 @@ func (state *State) bootstrapSporkInfo(root protocol.Snapshot) func(*badger.Txn) return fmt.Errorf("could not insert spork ID: %w", err) } + sporkRootBlockHeight, err := params.SporkRootBlockHeight() + if err != nil { + return fmt.Errorf("could not get spork root block height: %w", err) + } + err = operation.InsertSporkRootBlockHeight(sporkRootBlockHeight)(tx) + if err != nil { + return fmt.Errorf("could not insert spork root block height: %w", err) + } + version, err := params.ProtocolVersion() if err != nil { return fmt.Errorf("could not get protocol version: %w", err) @@ -434,6 +490,15 @@ func (state *State) bootstrapSporkInfo(root protocol.Snapshot) func(*badger.Txn) return fmt.Errorf("could not insert protocol version: %w", err) } + threshold, err := params.EpochCommitSafetyThreshold() + if err != nil { + return fmt.Errorf("could not get epoch commit safety threshold: %w", err) + } + err = operation.InsertEpochCommitSafetyThreshold(threshold)(tx) + if err != nil { + return fmt.Errorf("could not insert epoch commit safety threshold: %w", err) + } + return nil } } @@ -477,12 +542,17 @@ func OpenState( if err != nil { return nil, fmt.Errorf("failed to update epoch metrics: %w", err) } + // populate the protocol state cache + err = state.populateCache() + if err != nil { + return nil, fmt.Errorf("failed to populate cache: %w", err) + } return state, nil } func (state *State) Params() protocol.Params { - return &Params{state: state} + return Params{state: state} } func (state *State) Sealed() protocol.Snapshot { @@ -511,10 +581,10 @@ func (state *State) AtHeight(height uint64) protocol.Snapshot { // retrieve the block ID for the finalized height var blockID flow.Identifier err := state.db.View(operation.LookupBlockHeight(height, &blockID)) - if errors.Is(err, storage.ErrNotFound) { - return invalid.NewSnapshotf("unknown finalized height %d: %w", height, statepkg.ErrUnknownSnapshotReference) - } if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return invalid.NewSnapshotf("unknown finalized height %d: %w", height, statepkg.ErrUnknownSnapshotReference) + } // critical storage error return invalid.NewSnapshotf("could not look up block by height: %w", err) } @@ -560,7 +630,7 @@ func newState( } } -// IsBootstrapped returns whether or not the database contains a bootstrapped state +// IsBootstrapped returns whether the database contains a bootstrapped state func IsBootstrapped(db *badger.DB) (bool, error) { var finalized uint64 err := db.View(operation.RetrieveFinalizedHeight(&finalized)) @@ -625,6 +695,23 @@ func (state *State) updateEpochMetrics(snap protocol.Snapshot) error { return nil } +// populateCache is used after opening or bootstrapping the state to populate the cache. +func (state *State) populateCache() error { + var rootHeight uint64 + err := state.db.View(operation.RetrieveRootHeight(&rootHeight)) + if err != nil { + return fmt.Errorf("could not read root block to populate cache: %w", err) + } + state.rootHeight = rootHeight + + sporkRootBlockHeight, err := state.Params().SporkRootBlockHeight() + if err != nil { + return fmt.Errorf("could not read spork root block height: %w", err) + } + state.sporkRootBlockHeight = sporkRootBlockHeight + return nil +} + // updateCommittedEpochFinalView updates the `committed_epoch_final_view` metric // based on the current epoch phase of the input snapshot. It should be called // at startup and during transitions between EpochSetup and EpochCommitted phases. @@ -664,8 +751,13 @@ func (state *State) updateCommittedEpochFinalView(snap protocol.Snapshot) error return nil } -func (m *State) isEpochEmergencyFallbackTriggered() (bool, error) { +// isEpochEmergencyFallbackTriggered checks whether epoch fallback has been globally triggered. +// Returns: +// * (true, nil) if epoch fallback is triggered +// * (false, nil) if epoch fallback is not triggered (including if the flag is not set) +// * (false, err) if an unexpected error occurs +func (state *State) isEpochEmergencyFallbackTriggered() (bool, error) { var triggered bool - err := m.db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) + err := state.db.View(operation.CheckEpochEmergencyFallbackTriggered(&triggered)) return triggered, err } diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index f73f5e43b2a..7619f6a612d 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -161,12 +161,12 @@ func TestBootstrapNonRoot(t *testing.T) { t.Run("with sealed block", func(t *testing.T) { after := snapshotAfter(t, rootSnapshot, func(state *bprotocol.FollowerState) protocol.Snapshot { block1 := unittest.BlockWithParentFixture(rootBlock) - buildBlock(t, state, block1) + buildFinalizedBlock(t, state, block1) receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1) block2 := unittest.BlockWithParentFixture(block1.Header) block2.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal1), unittest.WithReceipts(receipt1))) - buildBlock(t, state, block2) + buildFinalizedBlock(t, state, block2) child := unittest.BlockWithParentFixture(block2.Header) buildBlock(t, state, child) @@ -429,12 +429,15 @@ func snapshotAfter(t *testing.T, rootSnapshot protocol.Snapshot, f func(*bprotoc return after } -// buildBlock builds and marks valid the given block +// buildBlock extends the protocol state by the given block func buildBlock(t *testing.T, state protocol.MutableState, block *flow.Block) { - err := state.Extend(context.Background(), block) - require.NoError(t, err) - err = state.MarkValid(block.ID()) - require.NoError(t, err) + require.NoError(t, state.Extend(context.Background(), block)) +} + +// buildFinalizedBlock extends the protocol state by the given block and marks the block as finalized +func buildFinalizedBlock(t *testing.T, state protocol.MutableState, block *flow.Block) { + require.NoError(t, state.Extend(context.Background(), block)) + require.NoError(t, state.Finalize(context.Background(), block.ID())) } // assertSealingSegmentBlocksQueryable bootstraps the state with the given @@ -444,7 +447,13 @@ func assertSealingSegmentBlocksQueryableAfterBootstrap(t *testing.T, snapshot pr require.NoError(t, err) segment, err := state.Final().SealingSegment() - assert.NoError(t, err) + require.NoError(t, err) + + rootBlock, err := state.Params().Root() + require.NoError(t, err) + + // root block should be the highest block from the sealing segment + assert.Equal(t, segment.Highest().Header, rootBlock) // for each block in the sealing segment we should be able to query: // * Head diff --git a/state/protocol/badger/validity.go b/state/protocol/badger/validity.go index 5a0b5d5c86f..ac4909a6156 100644 --- a/state/protocol/badger/validity.go +++ b/state/protocol/badger/validity.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/onflow/flow-go/consensus/hotstuff/committees" - "github.com/onflow/flow-go/consensus/hotstuff/mocks" - "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/consensus/hotstuff/signature" "github.com/onflow/flow-go/consensus/hotstuff/validator" "github.com/onflow/flow-go/consensus/hotstuff/verification" @@ -13,30 +11,31 @@ import ( "github.com/onflow/flow-go/model/flow/factory" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/flow/order" - "github.com/onflow/flow-go/state" "github.com/onflow/flow-go/state/protocol" ) // isValidExtendingEpochSetup checks whether an epoch setup service being -// added to the state is valid. In addition to intrinsic validitym, we also +// added to the state is valid. In addition to intrinsic validity, we also // check that it is valid w.r.t. the previous epoch setup event, and the // current epoch status. +// Assumes all inputs besides extendingSetup are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status func isValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { - // We should only have a single epoch setup event per epoch. if status.NextEpoch.SetupID != flow.ZeroID { // true iff EpochSetup event for NEXT epoch was already included before - return protocol.NewInvalidServiceEventError("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) + return protocol.NewInvalidServiceEventErrorf("duplicate epoch setup service event: %x", status.NextEpoch.SetupID) } // The setup event should have the counter increased by one. if extendingSetup.Counter != activeSetup.Counter+1 { - return protocol.NewInvalidServiceEventError("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) + return protocol.NewInvalidServiceEventErrorf("next epoch setup has invalid counter (%d => %d)", activeSetup.Counter, extendingSetup.Counter) } // The first view needs to be exactly one greater than the current epoch final view if extendingSetup.FirstView != activeSetup.FinalView+1 { - return protocol.NewInvalidServiceEventError( + return protocol.NewInvalidServiceEventErrorf( "next epoch first view must be exactly 1 more than current epoch final view (%d != %d+1)", extendingSetup.FirstView, activeSetup.FinalView, @@ -44,19 +43,19 @@ func isValidExtendingEpochSetup(extendingSetup *flow.EpochSetup, activeSetup *fl } // Finally, the epoch setup event must contain all necessary information. - err := isValidEpochSetup(extendingSetup) + err := verifyEpochSetup(extendingSetup, true) if err != nil { - return protocol.NewInvalidServiceEventError("invalid epoch setup: %w", err) + return protocol.NewInvalidServiceEventErrorf("invalid epoch setup: %w", err) } return nil } -// isValidEpochSetup checks whether an epoch setup service event is intrinsically valid -func isValidEpochSetup(setup *flow.EpochSetup) error { - return verifyEpochSetup(setup, true) -} - +// verifyEpochSetup checks whether an `EpochSetup` event is syntactically correct. +// The boolean parameter `verifyNetworkAddress` controls, whether we want to permit +// nodes to share a networking address. +// This is a side-effect-free function. Any error return indicates that the +// EpochSetup event is not compliant with protocol rules. func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { // STEP 1: general sanity checks // the seed needs to be at least minimum length @@ -88,15 +87,14 @@ func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { } // there should be no nodes with zero weight - // TODO: we might want to remove the following as we generally want to allow nodes with - // zero weight in the protocol state. + // TODO: we might want to remove the following as we generally want to allow nodes with zero weight in the protocol state. for _, participant := range setup.Participants { if participant.Weight == 0 { return fmt.Errorf("node with zero weight (%x)", participant.NodeID) } } - // the participants must be ordered by canonical order + // the participants must be listed in canonical order if !setup.Participants.Sorted(order.Canonical) { return fmt.Errorf("participants are not canonically ordered") } @@ -147,51 +145,57 @@ func verifyEpochSetup(setup *flow.EpochSetup, verifyNetworkAddress bool) error { // added to the state is valid. In addition to intrinsic validity, we also // check that it is valid w.r.t. the previous epoch setup event, and the // current epoch status. +// Assumes all inputs besides extendingCommit are already validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the input service event is invalid to extend the currently active epoch status func isValidExtendingEpochCommit(extendingCommit *flow.EpochCommit, extendingSetup *flow.EpochSetup, activeSetup *flow.EpochSetup, status *flow.EpochStatus) error { // We should only have a single epoch commit event per epoch. if status.NextEpoch.CommitID != flow.ZeroID { // true iff EpochCommit event for NEXT epoch was already included before - return protocol.NewInvalidServiceEventError("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) + return protocol.NewInvalidServiceEventErrorf("duplicate epoch commit service event: %x", status.NextEpoch.CommitID) } // The epoch setup event needs to happen before the commit. if status.NextEpoch.SetupID == flow.ZeroID { - return protocol.NewInvalidServiceEventError("missing epoch setup for epoch commit") + return protocol.NewInvalidServiceEventErrorf("missing epoch setup for epoch commit") } // The commit event should have the counter increased by one. if extendingCommit.Counter != activeSetup.Counter+1 { - return protocol.NewInvalidServiceEventError("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) + return protocol.NewInvalidServiceEventErrorf("next epoch commit has invalid counter (%d => %d)", activeSetup.Counter, extendingCommit.Counter) } err := isValidEpochCommit(extendingCommit, extendingSetup) if err != nil { - return state.NewInvalidExtensionErrorf("invalid epoch commit: %s", err) + return protocol.NewInvalidServiceEventErrorf("invalid epoch commit: %s", err) } return nil } // isValidEpochCommit checks whether an epoch commit service event is intrinsically valid. +// Assumes the input flow.EpochSetup event has already been validated. +// Expected errors during normal operations: +// * protocol.InvalidServiceEventError if the EpochCommit is invalid func isValidEpochCommit(commit *flow.EpochCommit, setup *flow.EpochSetup) error { if len(setup.Assignments) != len(commit.ClusterQCs) { - return fmt.Errorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) + return protocol.NewInvalidServiceEventErrorf("number of clusters (%d) does not number of QCs (%d)", len(setup.Assignments), len(commit.ClusterQCs)) } if commit.Counter != setup.Counter { - return fmt.Errorf("inconsistent epoch counter between commit (%d) and setup (%d) events in same epoch", commit.Counter, setup.Counter) + return protocol.NewInvalidServiceEventErrorf("inconsistent epoch counter between commit (%d) and setup (%d) events in same epoch", commit.Counter, setup.Counter) } // make sure we have a valid DKG public key if commit.DKGGroupKey == nil { - return fmt.Errorf("missing DKG public group key") + return protocol.NewInvalidServiceEventErrorf("missing DKG public group key") } participants := setup.Participants.Filter(filter.IsValidDKGParticipant) if len(participants) != len(commit.DKGParticipantKeys) { - return fmt.Errorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) + return protocol.NewInvalidServiceEventErrorf("participant list (len=%d) does not match dkg key list (len=%d)", len(participants), len(commit.DKGParticipantKeys)) } return nil @@ -216,7 +220,7 @@ func IsValidRootSnapshot(snap protocol.Snapshot, verifyResultID bool) error { } highest := segment.Highest() // reference block of the snapshot - lowest := segment.Lowest() // last sealed block + lowest := segment.Sealed() // last sealed block highestID := highest.ID() lowestID := lowest.ID() @@ -303,11 +307,6 @@ func IsValidRootSnapshotQCs(snap protocol.Snapshot) error { // validateRootQC performs validation of root QC // Returns nil on success func validateRootQC(snap protocol.Snapshot) error { - rootBlock, err := snap.Head() - if err != nil { - return fmt.Errorf("could not get root block: %w", err) - } - identities, err := snap.Identities(filter.IsVotingConsensusCommitteeMember) if err != nil { return fmt.Errorf("could not get root snapshot identities: %w", err) @@ -323,15 +322,13 @@ func validateRootQC(snap protocol.Snapshot) error { return fmt.Errorf("could not get DKG for root snapshot: %w", err) } - hotstuffRootBlock := model.GenesisBlockFromFlow(rootBlock) committee, err := committees.NewStaticCommitteeWithDKG(identities, flow.Identifier{}, dkg) if err != nil { return fmt.Errorf("could not create static committee: %w", err) } verifier := verification.NewCombinedVerifier(committee, signature.NewConsensusSigDataPacker(committee)) - forks := &mocks.ForksReader{} - hotstuffValidator := validator.New(committee, forks, verifier) - err = hotstuffValidator.ValidateQC(rootQC, hotstuffRootBlock) + hotstuffValidator := validator.New(committee, verifier) + err = hotstuffValidator.ValidateQC(rootQC) if err != nil { return fmt.Errorf("could not validate root qc: %w", err) } @@ -341,18 +338,81 @@ func validateRootQC(snap protocol.Snapshot) error { // validateClusterQC performs QC validation of single collection cluster // Returns nil on success func validateClusterQC(cluster protocol.Cluster) error { - clusterRootBlock := model.GenesisBlockFromFlow(cluster.RootBlock().Header) - committee, err := committees.NewStaticCommittee(cluster.Members(), flow.Identifier{}, nil, nil) if err != nil { return fmt.Errorf("could not create static committee: %w", err) } verifier := verification.NewStakingVerifier() - forks := &mocks.ForksReader{} - hotstuffValidator := validator.New(committee, forks, verifier) - err = hotstuffValidator.ValidateQC(cluster.RootQC(), clusterRootBlock) + hotstuffValidator := validator.New(committee, verifier) + err = hotstuffValidator.ValidateQC(cluster.RootQC()) if err != nil { return fmt.Errorf("could not validate root qc: %w", err) } return nil } + +// ValidRootSnapshotContainsEntityExpiryRange performs a sanity check to make sure the +// root snapshot has enough history to encompass at least one full entity expiry window. +// Entities (in particular transactions and collections) may reference a block within +// the past `flow.DefaultTransactionExpiry` blocks, so a new node must begin with at least +// this many blocks worth of history leading up to the snapshot's root block. +// +// Currently, Access Nodes and Consensus Nodes require root snapshots passing this validator function. +// +// - Consensus Nodes because they process guarantees referencing past blocks +// - Access Nodes because they index transactions referencing past blocks +// +// One of the following conditions must be satisfied to pass this validation: +// 1. This is a snapshot build from a first block of spork +// -> there is no earlier history which transactions/collections could reference +// 2. This snapshot sealing segment contains at least one expiry window of blocks +// -> all possible reference blocks in future transactions/collections will be within the initial history. +// 3. This snapshot sealing segment includes the spork root block +// -> there is no earlier history which transactions/collections could reference +func ValidRootSnapshotContainsEntityExpiryRange(snapshot protocol.Snapshot) error { + isSporkRootSnapshot, err := protocol.IsSporkRootSnapshot(snapshot) + if err != nil { + return fmt.Errorf("could not check if root snapshot is a spork root snapshot: %w", err) + } + // Condition 1 satisfied + if isSporkRootSnapshot { + return nil + } + + head, err := snapshot.Head() + if err != nil { + return fmt.Errorf("could not query root snapshot head: %w", err) + } + + sporkRootBlockHeight, err := snapshot.Params().SporkRootBlockHeight() + if err != nil { + return fmt.Errorf("could not query spork root block height: %w", err) + } + + sealingSegment, err := snapshot.SealingSegment() + if err != nil { + return fmt.Errorf("could not query sealing segment: %w", err) + } + + sealingSegmentLength := uint64(len(sealingSegment.AllBlocks())) + transactionExpiry := uint64(flow.DefaultTransactionExpiry) + blocksInSpork := head.Height - sporkRootBlockHeight + 1 // range is inclusive on both ends + + // Condition 3: + // check if head.Height - sporkRootBlockHeight < flow.DefaultTransactionExpiry + // this is the case where we bootstrap early into the spork and there is simply not enough blocks + if blocksInSpork < transactionExpiry { + // the distance to spork root is less than transaction expiry, we need all blocks back to the spork root. + if sealingSegmentLength != blocksInSpork { + return fmt.Errorf("invalid root snapshot length, expecting exactly (%d), got (%d)", blocksInSpork, sealingSegmentLength) + } + } else { + // Condition 2: + // the distance to spork root is more than transaction expiry, we need at least `transactionExpiry` many blocks + if sealingSegmentLength < transactionExpiry { + return fmt.Errorf("invalid root snapshot length, expecting at least (%d), got (%d)", + transactionExpiry, sealingSegmentLength) + } + } + return nil +} diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index 518e56e88e6..2c0e3372e4b 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -21,7 +21,7 @@ func TestEpochSetupValidity(t *testing.T) { // set an invalid final view for the first epoch setup.FinalView = setup.FirstView - err := isValidEpochSetup(setup) + err := verifyEpochSetup(setup, true) require.Error(t, err) }) @@ -31,7 +31,7 @@ func TestEpochSetupValidity(t *testing.T) { // randomly shuffle the identities so they are not canonically ordered setup.Participants = setup.Participants.DeterministicShuffle(time.Now().UnixNano()) - err := isValidEpochSetup(setup) + err := verifyEpochSetup(setup, true) require.Error(t, err) }) @@ -42,7 +42,7 @@ func TestEpochSetupValidity(t *testing.T) { collector := participants.Filter(filter.HasRole(flow.RoleCollection))[0] setup.Assignments = append(setup.Assignments, []flow.Identifier{collector.NodeID}) - err := isValidEpochSetup(setup) + err := verifyEpochSetup(setup, true) require.Error(t, err) }) @@ -51,7 +51,7 @@ func TestEpochSetupValidity(t *testing.T) { setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) setup.RandomSource = unittest.SeedFixture(crypto.SeedMinLenDKG - 1) - err := isValidEpochSetup(setup) + err := verifyEpochSetup(setup, true) require.Error(t, err) }) } @@ -101,3 +101,47 @@ func TestBootstrapInvalidEpochCommit(t *testing.T) { require.Error(t, err) }) } + +// TestEntityExpirySnapshotValidation tests that we perform correct sanity checks when +// bootstrapping consensus nodes and access nodes we expect that we only bootstrap snapshots +// with sufficient history. +func TestEntityExpirySnapshotValidation(t *testing.T) { + t.Run("spork-root-snapshot", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("not-enough-history", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + rootSnapshot.Encodable().Head.Height += 10 // advance height to be not spork root snapshot + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.Error(t, err) + }) + t.Run("enough-history-spork-just-started", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot, but still lower than transaction expiry + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry / 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(int(flow.DefaultTransactionExpiry / 2)) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("enough-history-long-spork", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry * 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(int(flow.DefaultTransactionExpiry) - 1) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) + t.Run("more-history-than-needed", func(t *testing.T) { + rootSnapshot := unittest.RootSnapshotFixture(participants) + // advance height to be not spork root snapshot + rootSnapshot.Encodable().Head.Height += flow.DefaultTransactionExpiry * 2 + // add blocks to sealing segment + rootSnapshot.Encodable().SealingSegment.ExtraBlocks = unittest.BlockFixtures(flow.DefaultTransactionExpiry * 2) + err := ValidRootSnapshotContainsEntityExpiryRange(rootSnapshot) + require.NoError(t, err) + }) +} diff --git a/state/protocol/defaults.go b/state/protocol/defaults.go new file mode 100644 index 00000000000..320c897d638 --- /dev/null +++ b/state/protocol/defaults.go @@ -0,0 +1,21 @@ +package protocol + +import ( + "fmt" + + "github.com/onflow/flow-go/model/flow" +) + +// DefaultEpochCommitSafetyThreshold returns the default epoch commit safety +// threshold for each chain ID. Greater threshold values are generally safer, +// but require longer epochs and longer EpochCommit phases. See Params for +// more details on this value. +func DefaultEpochCommitSafetyThreshold(chain flow.ChainID) (uint64, error) { + switch chain { + case flow.Mainnet, flow.Testnet, flow.Sandboxnet: + return 1_000, nil + case flow.Localnet, flow.Benchnet, flow.BftTestnet, flow.Emulator: + return 100, nil + } + return 0, fmt.Errorf("unkown chain id %s", chain.String()) +} diff --git a/state/protocol/errors.go b/state/protocol/errors.go index 4fabb93f2d0..34120f7a71b 100644 --- a/state/protocol/errors.go +++ b/state/protocol/errors.go @@ -17,17 +17,25 @@ var ( // has not been set up yet. ErrNextEpochNotSetup = fmt.Errorf("next epoch has not yet been set up") - // ErrEpochNotCommitted is a sentinel error returned when the epoch has - // not been committed and information is queried that is only accessible + // ErrNextEpochNotCommitted is a sentinel error returned when the next epoch + // has not been committed and information is queried that is only accessible // in the EpochCommitted phase. - ErrEpochNotCommitted = fmt.Errorf("queried info from EpochCommit event before it was emitted") + ErrNextEpochNotCommitted = fmt.Errorf("queried info from EpochCommit event before it was emitted") // ErrSealingSegmentBelowRootBlock is a sentinel error returned for queries - // for a sealing segment below the root block. - ErrSealingSegmentBelowRootBlock = fmt.Errorf("cannot query sealing segment below root block") + // for a sealing segment below the root block (local history cutoff). + ErrSealingSegmentBelowRootBlock = fmt.Errorf("cannot construct sealing segment beyond locally known history") // ErrClusterNotFound is a sentinel error returns for queries for a cluster ErrClusterNotFound = fmt.Errorf("could not find cluster") + + // ErrMultipleSealsForSameHeight indicates that an (unordered) slice of seals + // contains two or more seals for the same block height (possibilities include + // duplicated seals or seals for different blocks at the same height). + ErrMultipleSealsForSameHeight = fmt.Errorf("multiple seals for same block height") + + // ErrDiscontinuousSeals indicates that an (unordered) slice of seals skips at least one block height. + ErrDiscontinuousSeals = fmt.Errorf("seals have discontinuity, i.e. they skip some block heights") ) type IdentityNotFoundError struct { @@ -80,10 +88,10 @@ func IsInvalidServiceEventError(err error) bool { return errors.As(err, &errInvalidServiceEventError) } -// NewInvalidServiceEventError returns an invalid service event error. Since all invalid +// NewInvalidServiceEventErrorf returns an invalid service event error. Since all invalid // service events indicate an invalid extension, the service event error is wrapped in // the invalid extension error at construction. -func NewInvalidServiceEventError(msg string, args ...interface{}) error { +func NewInvalidServiceEventErrorf(msg string, args ...interface{}) error { return state.NewInvalidExtensionErrorf( "cannot extend state with invalid service event: %w", InvalidServiceEventError{ @@ -91,3 +99,29 @@ func NewInvalidServiceEventError(msg string, args ...interface{}) error { }, ) } + +// UnfinalizedSealingSegmentError indicates that including unfinalized blocks +// in the sealing segment is illegal. +type UnfinalizedSealingSegmentError struct { + error +} + +func NewUnfinalizedSealingSegmentErrorf(msg string, args ...interface{}) error { + return UnfinalizedSealingSegmentError{ + error: fmt.Errorf(msg, args...), + } +} + +func (e UnfinalizedSealingSegmentError) Unwrap() error { + return e.error +} + +func (e UnfinalizedSealingSegmentError) Error() string { + return e.error.Error() +} + +// IsUnfinalizedSealingSegmentError returns true if err is of type UnfinalizedSealingSegmentError +func IsUnfinalizedSealingSegmentError(err error) bool { + var e UnfinalizedSealingSegmentError + return errors.As(err, &e) +} diff --git a/state/protocol/events.go b/state/protocol/events.go index d39f9f139d7..4b1230b2bad 100644 --- a/state/protocol/events.go +++ b/state/protocol/events.go @@ -8,8 +8,18 @@ import ( // can be propagated to other components via an implementation of this interface. // Collectively, these are referred to as "Protocol Events". // -// Protocol events are guaranteed to be delivered at least once. Subscribers must -// handle multiple deliveries. +// Protocol events are delivered immediately after the database transaction +// committing the corresponding state change completes successfully. +// This means that events are delivered exactly once, while the system is running. +// Events may not be delivered during crashes and restarts, but any missed events +// are guaranteed to be reflected in the Protocol State upon restarting. +// Components consuming protocol events which cannot tolerate missed events +// must implement initialization logic which accounts for any missed events. +// +// EXAMPLE: +// Suppose block A is finalized at height 100. If the BlockFinalized(A) event is +// dropped due to a crash, then when the node restarts, the latest finalized block +// in the Protocol State is guaranteed to be A. // // CAUTION: Protocol event subscriber callbacks are invoked synchronously in the // critical path of protocol state mutations. Most subscribers should immediately @@ -25,11 +35,14 @@ type Consumer interface { // of this callback must handle repeated calls for the same block. BlockFinalized(block *flow.Header) - // BlockProcessable is called when a correct block is encountered - // that is ready to be processed (i.e. it is connected to the finalized - // chain and its source of randomness is available). + // BlockProcessable is called when a correct block is encountered that is + // ready to be processed (i.e. it is connected to the finalized chain and + // its source of randomness is available). BlockProcessable is never emitted + // for the root block, as the root block is always processable. // Formally, this callback is informationally idempotent. I.e. the consumer // of this callback must handle repeated calls for the same block. + // TODO trigger this on block insertion (used to be on MarkValid) + // - don't trigger for root block or below BlockProcessable(block *flow.Header) // EpochTransition is called when we transition to a new epoch. This is @@ -75,4 +88,11 @@ type Consumer interface { // // NOTE: Only called once the phase transition has been finalized. EpochCommittedPhaseStarted(currentEpochCounter uint64, first *flow.Header) + + // EpochEmergencyFallbackTriggered is called when epoch fallback mode (EECC) is triggered. + // Since EECC is a permanent, spork-scoped state, this event is triggered only once. + // After this event is triggered, no further epoch transitions will occur, + // no further epoch phase transitions will occur, and no further epoch-related + // related protocol events (the events defined in this interface) will be emitted. + EpochEmergencyFallbackTriggered() } diff --git a/state/protocol/events/distributor.go b/state/protocol/events/distributor.go index a6135a8a39b..e483291fc2e 100644 --- a/state/protocol/events/distributor.go +++ b/state/protocol/events/distributor.go @@ -63,3 +63,11 @@ func (d *Distributor) EpochCommittedPhaseStarted(epoch uint64, first *flow.Heade sub.EpochCommittedPhaseStarted(epoch, first) } } + +func (d *Distributor) EpochEmergencyFallbackTriggered() { + d.mu.RLock() + defer d.mu.RUnlock() + for _, sub := range d.subscribers { + sub.EpochEmergencyFallbackTriggered() + } +} diff --git a/state/protocol/events/noop.go b/state/protocol/events/noop.go index c71e5375a31..2e6714c540c 100644 --- a/state/protocol/events/noop.go +++ b/state/protocol/events/noop.go @@ -14,17 +14,14 @@ func NewNoop() *Noop { return &Noop{} } -func (n Noop) BlockFinalized(block *flow.Header) { -} +func (n Noop) BlockFinalized(block *flow.Header) {} -func (n Noop) BlockProcessable(block *flow.Header) { -} +func (n Noop) BlockProcessable(block *flow.Header) {} -func (n Noop) EpochTransition(newEpoch uint64, first *flow.Header) { -} +func (n Noop) EpochTransition(newEpoch uint64, first *flow.Header) {} -func (n Noop) EpochSetupPhaseStarted(epoch uint64, first *flow.Header) { -} +func (n Noop) EpochSetupPhaseStarted(epoch uint64, first *flow.Header) {} -func (n Noop) EpochCommittedPhaseStarted(epoch uint64, first *flow.Header) { -} +func (n Noop) EpochCommittedPhaseStarted(epoch uint64, first *flow.Header) {} + +func (n Noop) EpochEmergencyFallbackTriggered() {} diff --git a/state/protocol/inmem/convert.go b/state/protocol/inmem/convert.go index 32e7d0350c5..cf26b1d99b0 100644 --- a/state/protocol/inmem/convert.go +++ b/state/protocol/inmem/convert.go @@ -16,7 +16,6 @@ import ( // one that can easily be serialized to disk or to network. // TODO error docs func FromSnapshot(from protocol.Snapshot) (*Snapshot, error) { - var ( snap EncodableSnapshot err error @@ -89,7 +88,6 @@ func FromSnapshot(from protocol.Snapshot) (*Snapshot, error) { // FromParams converts any protocol.GlobalParams to a memory-backed Params. // TODO error docs func FromParams(from protocol.GlobalParams) (*Params, error) { - var ( params EncodableParams err error @@ -103,10 +101,18 @@ func FromParams(from protocol.GlobalParams) (*Params, error) { if err != nil { return nil, fmt.Errorf("could not get spork id: %w", err) } + params.SporkRootBlockHeight, err = from.SporkRootBlockHeight() + if err != nil { + return nil, fmt.Errorf("could not get spork root block height: %w", err) + } params.ProtocolVersion, err = from.ProtocolVersion() if err != nil { return nil, fmt.Errorf("could not get protocol version: %w", err) } + params.EpochCommitSafetyThreshold, err = from.EpochCommitSafetyThreshold() + if err != nil { + return nil, fmt.Errorf("could not get protocol version: %w", err) + } return &Params{params}, nil } @@ -117,7 +123,6 @@ func FromParams(from protocol.GlobalParams) (*Params, error) { // * protocol.ErrNextEpochNotSetup - if the epoch represents a next epoch which has not been set up. // * state.ErrUnknownSnapshotReference - if the epoch is queried from an unresolvable snapshot. func FromEpoch(from protocol.Epoch) (*Epoch, error) { - var ( epoch EncodableEpoch err error @@ -157,7 +162,7 @@ func FromEpoch(from protocol.Epoch) (*Epoch, error) { // convert dkg dkg, err := from.DKG() // if this epoch hasn't been committed yet, return the epoch as-is - if errors.Is(err, protocol.ErrEpochNotCommitted) { + if errors.Is(err, protocol.ErrNextEpochNotCommitted) { return &Epoch{epoch}, nil } if err != nil { @@ -230,19 +235,24 @@ func ClusterFromEncodable(enc EncodableCluster) (*Cluster, error) { // root bootstrap state. This is used to bootstrap the protocol state for // genesis or post-spork states. func SnapshotFromBootstrapState(root *flow.Block, result *flow.ExecutionResult, seal *flow.Seal, qc *flow.QuorumCertificate) (*Snapshot, error) { - return SnapshotFromBootstrapStateWithProtocolVersion(root, result, seal, qc, flow.DefaultProtocolVersion) + version := flow.DefaultProtocolVersion + threshold, err := protocol.DefaultEpochCommitSafetyThreshold(root.Header.ChainID) + if err != nil { + return nil, fmt.Errorf("could not get default epoch commit safety threshold: %w", err) + } + return SnapshotFromBootstrapStateWithParams(root, result, seal, qc, version, threshold) } -// SnapshotFromBootstrapStateWithProtocolVersion is SnapshotFromBootstrapState +// SnapshotFromBootstrapStateWithParams is SnapshotFromBootstrapState // with a caller-specified protocol version. -func SnapshotFromBootstrapStateWithProtocolVersion( +func SnapshotFromBootstrapStateWithParams( root *flow.Block, result *flow.ExecutionResult, seal *flow.Seal, qc *flow.QuorumCertificate, - version uint, + protocolVersion uint, + epochCommitSafetyThreshold uint64, ) (*Snapshot, error) { - setup, ok := result.ServiceEvents[0].Event.(*flow.EpochSetup) if !ok { return nil, fmt.Errorf("invalid setup event type (%T)", result.ServiceEvents[0].Event) @@ -282,9 +292,11 @@ func SnapshotFromBootstrapStateWithProtocolVersion( } params := EncodableParams{ - ChainID: root.Header.ChainID, // chain ID must match the root block - SporkID: root.ID(), // use root block ID as the unique spork identifier - ProtocolVersion: version, // major software version for this spork + ChainID: root.Header.ChainID, // chain ID must match the root block + SporkID: root.ID(), // use root block ID as the unique spork identifier + SporkRootBlockHeight: root.Header.Height, // use root block height as the spork root block height + ProtocolVersion: protocolVersion, // major software version for this spork + EpochCommitSafetyThreshold: epochCommitSafetyThreshold, // see protocol.Params for details } snap := SnapshotFromEncodable(EncodableSnapshot{ @@ -297,6 +309,7 @@ func SnapshotFromBootstrapStateWithProtocolVersion( ExecutionResults: flow.ExecutionResultList{result}, LatestSeals: map[flow.Identifier]flow.Identifier{root.ID(): seal.ID()}, FirstSeal: seal, + ExtraBlocks: make([]*flow.Block, 0), }, QuorumCertificate: qc, Phase: flow.EpochPhaseStaking, diff --git a/state/protocol/inmem/encodable.go b/state/protocol/inmem/encodable.go index bb7fba09d97..c52d85ae3e0 100644 --- a/state/protocol/inmem/encodable.go +++ b/state/protocol/inmem/encodable.go @@ -64,7 +64,9 @@ type EncodableCluster struct { // EncodableParams is the encoding format for protocol.GlobalParams type EncodableParams struct { - ChainID flow.ChainID - SporkID flow.Identifier - ProtocolVersion uint + ChainID flow.ChainID + SporkID flow.Identifier + SporkRootBlockHeight uint64 + ProtocolVersion uint + EpochCommitSafetyThreshold uint64 } diff --git a/state/protocol/inmem/epoch.go b/state/protocol/inmem/epoch.go index 85d9ce71538..9843fd52e6c 100644 --- a/state/protocol/inmem/epoch.go +++ b/state/protocol/inmem/epoch.go @@ -43,12 +43,12 @@ func (e Epoch) DKG() (protocol.DKG, error) { if e.enc.DKG != nil { return DKG{*e.enc.DKG}, nil } - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } func (e Epoch) Cluster(i uint) (protocol.Cluster, error) { if e.enc.Clusters == nil { - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } if i >= uint(len(e.enc.Clusters)) { @@ -59,7 +59,7 @@ func (e Epoch) Cluster(i uint) (protocol.Cluster, error) { func (e Epoch) ClusterByChainID(chainID flow.ChainID) (protocol.Cluster, error) { if e.enc.Clusters == nil { - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } for _, cluster := range e.enc.Clusters { @@ -144,15 +144,15 @@ func ClusteringFromSetupEvent(setupEvent *flow.EpochSetup) (flow.ClusterList, er } func (es *setupEpoch) Cluster(_ uint) (protocol.Cluster, error) { - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } func (es *setupEpoch) ClusterByChainID(_ flow.ChainID) (protocol.Cluster, error) { - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } func (es *setupEpoch) DKG() (protocol.DKG, error) { - return nil, protocol.ErrEpochNotCommitted + return nil, protocol.ErrNextEpochNotCommitted } func (es *setupEpoch) RandomSource() ([]byte, error) { diff --git a/state/protocol/inmem/params.go b/state/protocol/inmem/params.go index 9c8fc62e7ad..64e00eb4eda 100644 --- a/state/protocol/inmem/params.go +++ b/state/protocol/inmem/params.go @@ -16,6 +16,14 @@ func (p Params) SporkID() (flow.Identifier, error) { return p.enc.SporkID, nil } +func (p Params) SporkRootBlockHeight() (uint64, error) { + return p.enc.SporkRootBlockHeight, nil +} + func (p Params) ProtocolVersion() (uint, error) { return p.enc.ProtocolVersion, nil } + +func (p Params) EpochCommitSafetyThreshold() (uint64, error) { + return p.enc.EpochCommitSafetyThreshold, nil +} diff --git a/state/protocol/inmem/snapshot.go b/state/protocol/inmem/snapshot.go index 9b82b41ed3b..8ad5727bfb8 100644 --- a/state/protocol/inmem/snapshot.go +++ b/state/protocol/inmem/snapshot.go @@ -58,11 +58,6 @@ func (s Snapshot) Descendants() ([]flow.Identifier, error) { return nil, nil } -func (s Snapshot) ValidDescendants() ([]flow.Identifier, error) { - // canonical snapshots don't have any descendants - return nil, nil -} - func (s Snapshot) Phase() (flow.EpochPhase, error) { return s.enc.Phase, nil } diff --git a/state/protocol/invalid/params.go b/state/protocol/invalid/params.go index 5b2ae2aae9a..a131d4517a8 100644 --- a/state/protocol/invalid/params.go +++ b/state/protocol/invalid/params.go @@ -9,14 +9,22 @@ type Params struct { err error } -func (p *Params) ChainID() (flow.ChainID, error) { +func (p Params) ChainID() (flow.ChainID, error) { return "", p.err } -func (p *Params) SporkID() (flow.Identifier, error) { +func (p Params) SporkID() (flow.Identifier, error) { return flow.ZeroID, p.err } -func (p *Params) ProtocolVersion() (uint, error) { +func (p Params) SporkRootBlockHeight() (uint64, error) { + return 0, p.err +} + +func (p Params) ProtocolVersion() (uint, error) { + return 0, p.err +} + +func (p Params) EpochCommitSafetyThreshold() (uint64, error) { return 0, p.err } diff --git a/state/protocol/invalid/snapshot.go b/state/protocol/invalid/snapshot.go index 9b3bdddcf7f..ab54103c191 100644 --- a/state/protocol/invalid/snapshot.go +++ b/state/protocol/invalid/snapshot.go @@ -68,14 +68,10 @@ func (u *Snapshot) Descendants() ([]flow.Identifier, error) { return nil, u.err } -func (u *Snapshot) ValidDescendants() ([]flow.Identifier, error) { - return nil, u.err -} - func (u *Snapshot) RandomSource() ([]byte, error) { return nil, u.err } func (u *Snapshot) Params() protocol.GlobalParams { - return &Params{u.err} + return Params{u.err} } diff --git a/state/protocol/mock/consumer.go b/state/protocol/mock/consumer.go index 4177e8a1537..9978547f6d1 100644 --- a/state/protocol/mock/consumer.go +++ b/state/protocol/mock/consumer.go @@ -27,6 +27,11 @@ func (_m *Consumer) EpochCommittedPhaseStarted(currentEpochCounter uint64, first _m.Called(currentEpochCounter, first) } +// EpochEmergencyFallbackTriggered provides a mock function with given fields: +func (_m *Consumer) EpochEmergencyFallbackTriggered() { + _m.Called() +} + // EpochSetupPhaseStarted provides a mock function with given fields: currentEpochCounter, first func (_m *Consumer) EpochSetupPhaseStarted(currentEpochCounter uint64, first *flow.Header) { _m.Called(currentEpochCounter, first) diff --git a/state/protocol/mock/global_params.go b/state/protocol/mock/global_params.go index d559c68f0c0..4ecf14ed03f 100644 --- a/state/protocol/mock/global_params.go +++ b/state/protocol/mock/global_params.go @@ -33,6 +33,27 @@ func (_m *GlobalParams) ChainID() (flow.ChainID, error) { return r0, r1 } +// EpochCommitSafetyThreshold provides a mock function with given fields: +func (_m *GlobalParams) EpochCommitSafetyThreshold() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ProtocolVersion provides a mock function with given fields: func (_m *GlobalParams) ProtocolVersion() (uint, error) { ret := _m.Called() @@ -77,6 +98,27 @@ func (_m *GlobalParams) SporkID() (flow.Identifier, error) { return r0, r1 } +// SporkRootBlockHeight provides a mock function with given fields: +func (_m *GlobalParams) SporkRootBlockHeight() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewGlobalParams interface { mock.TestingT Cleanup(func()) diff --git a/state/protocol/mock/instance_params.go b/state/protocol/mock/instance_params.go index 61b5c877af2..18f7cc27032 100644 --- a/state/protocol/mock/instance_params.go +++ b/state/protocol/mock/instance_params.go @@ -12,6 +12,27 @@ type InstanceParams struct { mock.Mock } +// EpochFallbackTriggered provides a mock function with given fields: +func (_m *InstanceParams) EpochFallbackTriggered() (bool, error) { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Root provides a mock function with given fields: func (_m *InstanceParams) Root() (*flow.Header, error) { ret := _m.Called() diff --git a/state/protocol/mock/mutable_state.go b/state/protocol/mock/mutable_state.go index 5e886a7e6f3..6680aaf25c1 100644 --- a/state/protocol/mock/mutable_state.go +++ b/state/protocol/mock/mutable_state.go @@ -92,20 +92,6 @@ func (_m *MutableState) Finalize(ctx context.Context, blockID flow.Identifier) e return r0 } -// MarkValid provides a mock function with given fields: blockID -func (_m *MutableState) MarkValid(blockID flow.Identifier) error { - ret := _m.Called(blockID) - - var r0 error - if rf, ok := ret.Get(0).(func(flow.Identifier) error); ok { - r0 = rf(blockID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Params provides a mock function with given fields: func (_m *MutableState) Params() protocol.Params { ret := _m.Called() diff --git a/state/protocol/mock/params.go b/state/protocol/mock/params.go index 5a6398e769e..000140f5d42 100644 --- a/state/protocol/mock/params.go +++ b/state/protocol/mock/params.go @@ -33,6 +33,48 @@ func (_m *Params) ChainID() (flow.ChainID, error) { return r0, r1 } +// EpochCommitSafetyThreshold provides a mock function with given fields: +func (_m *Params) EpochCommitSafetyThreshold() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// EpochFallbackTriggered provides a mock function with given fields: +func (_m *Params) EpochFallbackTriggered() (bool, error) { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ProtocolVersion provides a mock function with given fields: func (_m *Params) ProtocolVersion() (uint, error) { ret := _m.Called() @@ -123,6 +165,27 @@ func (_m *Params) SporkID() (flow.Identifier, error) { return r0, r1 } +// SporkRootBlockHeight provides a mock function with given fields: +func (_m *Params) SporkRootBlockHeight() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + type mockConstructorTestingTNewParams interface { mock.TestingT Cleanup(func()) diff --git a/state/protocol/mock/snapshot.go b/state/protocol/mock/snapshot.go index eb22a227af9..4245913e3e0 100644 --- a/state/protocol/mock/snapshot.go +++ b/state/protocol/mock/snapshot.go @@ -283,29 +283,6 @@ func (_m *Snapshot) SealingSegment() (*flow.SealingSegment, error) { return r0, r1 } -// ValidDescendants provides a mock function with given fields: -func (_m *Snapshot) ValidDescendants() ([]flow.Identifier, error) { - ret := _m.Called() - - var r0 []flow.Identifier - if rf, ok := ret.Get(0).(func() []flow.Identifier); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.Identifier) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - type mockConstructorTestingTNewSnapshot interface { mock.TestingT Cleanup(func()) diff --git a/state/protocol/params.go b/state/protocol/params.go index a012991e67e..2c65ae73690 100644 --- a/state/protocol/params.go +++ b/state/protocol/params.go @@ -20,12 +20,21 @@ type InstanceParams interface { // Root returns the root header of the current protocol state. This will be // the head of the protocol state snapshot used to bootstrap this state and // may differ from node to node for the same protocol state. + // No errors are expected during normal operation. Root() (*flow.Header, error) // Seal returns the root block seal of the current protocol state. This will be // the seal for the root block used to bootstrap this state and may differ from // node to node for the same protocol state. + // No errors are expected during normal operation. Seal() (*flow.Seal, error) + + // EpochFallbackTriggered returns whether epoch fallback mode (EECC) has been triggered. + // EECC is a permanent, spork-scoped state which is triggered when the next + // epoch fails to be committed in the allocated time. Once EECC is triggered, + // it will remain in effect until the next spork. + // No errors are expected during normal operation. + EpochFallbackTriggered() (bool, error) } // GlobalParams represents protocol state parameters that do not vary between instances. @@ -35,14 +44,71 @@ type GlobalParams interface { // ChainID returns the chain ID for the current Flow network. The chain ID // uniquely identifies a Flow network in perpetuity across epochs and sporks. + // No errors are expected during normal operation. ChainID() (flow.ChainID, error) // SporkID returns the unique identifier for this network within the current spork. // This ID is determined at the beginning of a spork during bootstrapping and is // part of the root protocol state snapshot. + // No errors are expected during normal operation. SporkID() (flow.Identifier, error) + // SporkRootBlockHeight returns the height of the spork's root block. + // This value is determined at the beginning of a spork during bootstrapping. + // If node uses a sealing segment for bootstrapping then this value will be carried over + // as part of snapshot. + // No errors are expected during normal operation. + SporkRootBlockHeight() (uint64, error) + // ProtocolVersion returns the protocol version, the major software version // of the protocol software. + // No errors are expected during normal operation. ProtocolVersion() (uint, error) + + // EpochCommitSafetyThreshold defines a deadline for sealing the EpochCommit + // service event near the end of each epoch - the "epoch commitment deadline". + // Given a safety threshold t, the deadline for an epoch with final view f is: + // Epoch Commitment Deadline: d=f-t + // + // DEFINITION: + // This deadline is used to determine when to trigger epoch emergency fallback mode. + // Epoch Emergency Fallback mode is triggered when the EpochCommit service event + // fails to be sealed. + // + // Example: A service event is emitted in block A. The seal for A is included in C. + // A<-B(RA)<-C(SA)<-...<-R + // + // A service event S is considered sealed w.r.t. a reference block R if: + // * S was emitted during execution of some block A, s.t. A is an ancestor of R + // * The seal for block A was included in some block C, s.t C is an ancestor of R + // + // When we finalize the first block B with B.View >= d: + // HAPPY PATH: If an EpochCommit service event has been sealed w.r.t. B, no action is taken. + // FALLBACK PATH: If no EpochCommit service event has been sealed w.r.t. B, epoch fallback mode (EECC) is triggered. + // + // CONTEXT: + // The epoch commitment deadline exists to ensure that all nodes agree on + // whether epoch fallback mode is triggered for a particular epoch, before + // the epoch actually ends. Although the use of this deadline DOES NOT + // guarantee these properties, it is a simpler way to assure them with high + // likelihood, given reasonable configuration. + // In particular, all nodes will agree about EECC being triggered (or not) + // if at least one block with view in [d, f] is finalized - in other words + // at least one block is finalized after the epoch commitment deadline, and + // before the next epoch begins. + // + // When selecting a threshold value, ensure: + // * The deadline is after the end of the DKG, with enough buffer between + // the two that the EpochCommit event is overwhelmingly likely to be emitted + // before the deadline, if it is emitted at all. + // * The buffer between the deadline and the final view of the epoch is large + // enough that the network is overwhelming likely to finalize at least one + // block with a view in this range + // + // /- Epoch Commitment Deadline + // EPOCH N v EPOCH N+1 + // ...------------|------||-----... + // + // No errors are expected during normal operation. + EpochCommitSafetyThreshold() (uint64, error) } diff --git a/state/protocol/seed/customizers.go b/state/protocol/seed/customizers.go index ea3700aa35d..0262c8661d3 100644 --- a/state/protocol/seed/customizers.go +++ b/state/protocol/seed/customizers.go @@ -3,7 +3,7 @@ package seed import "encoding/binary" // list of customizers used for different sub-protocol PRNGs. -// These customizers help instanciate different PRNGs from the +// These customizers help instantiate different PRNGs from the // same source of randomness. var ( diff --git a/state/protocol/snapshot.go b/state/protocol/snapshot.go index 4064685405d..2130d10620b 100644 --- a/state/protocol/snapshot.go +++ b/state/protocol/snapshot.go @@ -85,28 +85,24 @@ type Snapshot interface { // For all other snapshots, the sealing segment contains at least 2 blocks. // // NOTE 3: It is often the case that a block inside the segment will contain - // an execution receipt in it's payload that references an execution result + // an execution receipt in its payload that references an execution result // missing from the payload. These missing execution results are stored on the // flow.SealingSegment.ExecutionResults field. - // TODO document error returns + // Expected errors during normal operations: + // - protocol.ErrSealingSegmentBelowRootBlock if sealing segment would stretch beyond the node's local history cut-off + // - protocol.UnfinalizedSealingSegmentError if sealing segment would contain unfinalized blocks (including orphaned blocks) SealingSegment() (*flow.SealingSegment, error) - // Descendants returns the IDs of all descendants of the Head block. The IDs - // are ordered such that parents are included before their children. These - // are NOT guaranteed to have been validated by HotStuff. - // TODO document error returns + // Descendants returns the IDs of all descendants of the Head block. + // The IDs are ordered such that parents are included before their children. + // Since all blocks are fully validated before being inserted to the state, + // all returned blocks are validated. + // No errors are expected under normal operation. Descendants() ([]flow.Identifier, error) - // ValidDescendants returns the IDs of all descendants of the Head block. - // The IDs are ordered such that parents are included before their children. Includes - // only blocks that have been validated by HotStuff. - // TODO document error returns - ValidDescendants() ([]flow.Identifier, error) - - // RandomSource returns the source of randomness derived from the - // Head block. + // RandomSource returns the source of randomness derived from the Head block. // NOTE: not to be confused with the epoch source of randomness! - // error returns: + // Error returns: // * NoValidChildBlockError indicates that no valid child block is known // (which contains the block's source of randomness) // * unexpected errors should be considered symptoms of internal bugs diff --git a/state/protocol/state.go b/state/protocol/state.go index e89695a8d4b..94bcb41573a 100644 --- a/state/protocol/state.go +++ b/state/protocol/state.go @@ -42,9 +42,9 @@ type MutableState interface { // Extend introduces the block with the given ID into the persistent // protocol state without modifying the current finalized state. It allows // us to execute fork-aware queries against ambiguous protocol state, while - // still checking that the given block is a valid extension of the protocol - // state. Depending on implementation it might be a lighter version that checks only - // block header. + // still checking that the given block is a valid extension of the protocol state. + // Depending on implementation it might be a lighter version that checks only block header. + // The candidate block must have passed HotStuff validation before being passed to Extend. // Expected errors during normal operations: // * state.OutdatedExtensionError if the candidate block is outdated (e.g. orphaned) // * state.InvalidExtensionError if the candidate block is invalid @@ -58,12 +58,4 @@ type MutableState interface { // forwards the pointer to the latest finalized state. // TODO error docs Finalize(ctx context.Context, blockID flow.Identifier) error - - // MarkValid marks the block header with the given block hash as valid. - // At this level, we can only mark one block at a time as valid. This - // implies that the parent of the block to be marked as valid - // has to be already valid. - // It modifies the persistent immutable protocol state accordingly. - // TODO error docs - MarkValid(blockID flow.Identifier) error } diff --git a/state/protocol/util.go b/state/protocol/util.go index 8e9e14f8eab..7807a55edee 100644 --- a/state/protocol/util.go +++ b/state/protocol/util.go @@ -1,11 +1,14 @@ package protocol import ( + "errors" "fmt" + "math" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module/signature" + "github.com/onflow/flow-go/storage" ) // IsNodeAuthorizedAt returns whether the node with the given ID is a valid @@ -62,14 +65,28 @@ func CheckNodeStatusAt(snapshot Snapshot, id flow.Identifier, checks ...flow.Ide // IsSporkRootSnapshot returns whether the given snapshot is the state snapshot // representing the initial state for a spork. func IsSporkRootSnapshot(snapshot Snapshot) (bool, error) { - segment, err := snapshot.SealingSegment() + sporkRootBlockHeight, err := snapshot.Params().SporkRootBlockHeight() + if err != nil { + return false, fmt.Errorf("could not get snapshot root block height: %w", err) + } + head, err := snapshot.Head() if err != nil { return false, fmt.Errorf("could not get snapshot head: %w", err) } - if len(segment.Blocks) > 1 { - // spork root snapshots uniquely have only one block in the sealing segment + return head.Height == sporkRootBlockHeight, nil +} + +// PreviousEpochExists returns whether the previous epoch exists w.r.t. the given +// state snapshot. +// No errors are expected during normal operation. +func PreviousEpochExists(snap Snapshot) (bool, error) { + _, err := snap.Epochs().Previous().Counter() + if errors.Is(err, ErrNoPreviousEpoch) { return false, nil } + if err != nil { + return false, fmt.Errorf("unexpected error checking previous epoch exists: %w", err) + } return true, nil } @@ -98,3 +115,53 @@ func FindGuarantors(state State, guarantee *flow.CollectionGuarantee) ([]flow.Id return guarantorIDs, nil } + +// OrderedSeals returns the seals in the input payload in ascending height order. +// The Flow protocol has a variety of validity rules for `payload`. While we do not verify +// payload validity in this function, the implementation is optimized for valid payloads, +// where the heights of the sealed blocks form a continuous integer sequence (no gaps). +// Per convention ['Vacuous Truth'], an empty set of seals is considered to be +// ordered. Hence, if `payload.Seals` is empty, we return (nil, nil). +// Expected Error returns during normal operations: +// - ErrMultipleSealsForSameHeight in case there are seals repeatedly sealing block at the same height +// - ErrDiscontinuousSeals in case there are height-gaps in the sealed blocks +// - storage.ErrNotFound if any of the seals references an unknown block +func OrderedSeals(payload *flow.Payload, headers storage.Headers) ([]*flow.Seal, error) { + numSeals := uint64(len(payload.Seals)) + if numSeals == 0 { + return nil, nil + } + heights := make([]uint64, numSeals) + minHeight := uint64(math.MaxUint64) + for i, seal := range payload.Seals { + header, err := headers.ByBlockID(seal.BlockID) + if err != nil { + return nil, fmt.Errorf("could not get block (id=%x) for seal: %w", seal.BlockID, err) // storage.ErrNotFound or exception + } + heights[i] = header.Height + if header.Height < minHeight { + minHeight = header.Height + } + } + // As seals in a valid payload must have consecutive heights, we can populate + // the ordered output by shifting by minHeight. + seals := make([]*flow.Seal, numSeals) + for i, seal := range payload.Seals { + idx := heights[i] - minHeight + // (0) Per construction, `minHeight` is the smallest value in the `heights` slice. Hence, `idx ≥ 0` + // (1) But if there are gaps in the heights of the sealed blocks (byzantine inputs), + // `idx` may reach/exceed `numSeals`. In this case, we respond with a `ErrDiscontinuousSeals`. + // Thereby this function can tolerate all inputs without producing an 'out of range' panic. + // (2) In case of duplicates _and_ gaps, we might overwrite elements in `seals` while leaving + // other elements as `nil`. We avoid the edge case of leaving `nil` elements in our output, + // and instead, reject the input with an `ErrMultipleSealsForSameHeight` + if idx >= numSeals { + return nil, fmt.Errorf("sealed blocks' heights (unordered) %v : %w", heights, ErrDiscontinuousSeals) + } + if seals[idx] != nil { + return nil, fmt.Errorf("duplicate seal for block height %d: %w", heights[i], ErrMultipleSealsForSameHeight) + } + seals[idx] = seal + } + return seals, nil +} diff --git a/state/protocol/util/testing.go b/state/protocol/util/testing.go index 3a523111870..313adec9e85 100644 --- a/state/protocol/util/testing.go +++ b/state/protocol/util/testing.go @@ -152,6 +152,21 @@ func RunWithFullProtocolStateAndConsumer(t testing.TB, rootSnapshot protocol.Sna }) } +func RunWithFullProtocolStateAndMetricsAndConsumer(t testing.TB, rootSnapshot protocol.Snapshot, metrics module.ComplianceMetrics, consumer protocol.Consumer, f func(*badger.DB, *pbadger.MutableState)) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + tracer := trace.NewNoopTracer() + headers, _, seals, index, payloads, blocks, setups, commits, statuses, results := util.StorageLayer(t, db) + state, err := pbadger.Bootstrap(metrics, db, headers, seals, results, blocks, setups, commits, statuses, rootSnapshot) + require.NoError(t, err) + receiptValidator := MockReceiptValidator() + sealValidator := MockSealValidator(seals) + mockTimer := MockBlockTimer() + fullState, err := pbadger.NewFullConsensusState(state, index, payloads, tracer, consumer, mockTimer, receiptValidator, sealValidator) + require.NoError(t, err) + f(db, fullState) + }) +} + func RunWithFollowerProtocolStateAndHeaders(t testing.TB, rootSnapshot protocol.Snapshot, f func(*badger.DB, *pbadger.FollowerState, storage.Headers, storage.Index)) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { metrics := metrics.NewNoopCollector() diff --git a/state/protocol/util_test.go b/state/protocol/util_test.go index c0c13c8f077..7858f5767b7 100644 --- a/state/protocol/util_test.go +++ b/state/protocol/util_test.go @@ -1,12 +1,18 @@ package protocol_test import ( + "errors" + "math/rand" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + storagemock "github.com/onflow/flow-go/storage/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -20,9 +26,78 @@ func TestIsSporkRootSnapshot(t *testing.T) { t.Run("other snapshot", func(t *testing.T) { snapshot := unittest.RootSnapshotFixture(unittest.IdentityListFixture(10, unittest.WithAllRoles())) - snapshot.Encodable().SealingSegment.Blocks = unittest.BlockFixtures(5) + snapshot.Encodable().Head.Height += 1 // modify head height to break equivalence with spork root block height isSporkRoot, err := protocol.IsSporkRootSnapshot(snapshot) require.NoError(t, err) assert.False(t, isSporkRoot) }) } + +// TestOrderedSeals tests that protocol.OrderedSeals returns a list of ordered seals for a payload. +func TestOrderedSeals(t *testing.T) { + t.Run("empty payload", func(t *testing.T) { + payload := flow.EmptyPayload() + headers := storagemock.NewHeaders(t) + + ordered, err := protocol.OrderedSeals(&payload, headers) + require.NoError(t, err) + require.Empty(t, ordered) + }) + t.Run("block not found error", func(t *testing.T) { + headers := storagemock.NewHeaders(t) + seals := unittest.Seal.Fixtures(10) + payload := unittest.PayloadFixture(unittest.WithSeals(seals...)) + headers.On("ByBlockID", mock.Anything).Return(nil, storage.ErrNotFound) + + ordered, err := protocol.OrderedSeals(&payload, headers) + require.ErrorIs(t, err, storage.ErrNotFound) + require.Empty(t, ordered) + }) + t.Run("unexpected error", func(t *testing.T) { + headers := storagemock.NewHeaders(t) + seals := unittest.Seal.Fixtures(10) + payload := unittest.PayloadFixture(unittest.WithSeals(seals...)) + exception := errors.New("exception") + headers.On("ByBlockID", mock.Anything).Return(nil, exception) + + ordered, err := protocol.OrderedSeals(&payload, headers) + require.ErrorIs(t, err, exception) + require.Empty(t, ordered) + }) + t.Run("already ordered", func(t *testing.T) { + headers := storagemock.NewHeaders(t) + + blocks := unittest.ChainFixtureFrom(10, unittest.BlockHeaderFixture()) + seals := unittest.Seal.Fixtures(10) + for i, seal := range seals { + seal.BlockID = blocks[i].ID() + headers.On("ByBlockID", seal.BlockID).Return(blocks[i].Header, nil) + } + payload := unittest.PayloadFixture(unittest.WithSeals(seals...)) + + ordered, err := protocol.OrderedSeals(&payload, headers) + require.NoError(t, err) + require.Equal(t, seals, ordered) + }) + t.Run("unordered", func(t *testing.T) { + headers := storagemock.NewHeaders(t) + + blocks := unittest.ChainFixtureFrom(10, flow.Genesis(flow.Localnet).Header) + orderedSeals := unittest.Seal.Fixtures(len(blocks)) + for i, seal := range orderedSeals { + seal.BlockID = blocks[i].ID() + headers.On("ByBlockID", seal.BlockID).Return(blocks[i].Header, nil) + } + unorderedSeals := make([]*flow.Seal, len(orderedSeals)) + copy(unorderedSeals, orderedSeals) + // randomly re-order seals + rand.Shuffle(len(unorderedSeals), func(i, j int) { + unorderedSeals[i], unorderedSeals[j] = unorderedSeals[j], unorderedSeals[i] + }) + payload := unittest.PayloadFixture(unittest.WithSeals(unorderedSeals...)) + + ordered, err := protocol.OrderedSeals(&payload, headers) + require.NoError(t, err) + require.Equal(t, orderedSeals, ordered) + }) +} diff --git a/storage/badger/blocks.go b/storage/badger/blocks.go index 9c4e801a308..cc0a35e3acd 100644 --- a/storage/badger/blocks.go +++ b/storage/badger/blocks.go @@ -35,7 +35,7 @@ func (b *Blocks) StoreTx(block *flow.Block) func(*transaction.Tx) error { return func(tx *transaction.Tx) error { err := b.headers.storeTx(block.Header)(tx) if err != nil { - return fmt.Errorf("could not store header: %w", err) + return fmt.Errorf("could not store header %v: %w", block.Header.ID(), err) } err = b.payloads.storeTx(block.ID(), block.Payload)(tx) if err != nil { diff --git a/storage/badger/cache.go b/storage/badger/cache.go index 0364ea39378..d53cccb8131 100644 --- a/storage/badger/cache.go +++ b/storage/badger/cache.go @@ -80,7 +80,8 @@ func newCache(collector module.CacheMetrics, resourceName string, options ...fun } // Get will try to retrieve the resource from cache first, and then from the -// injected +// injected. During normal operations, the following error returns are expected: +// - `storage.ErrNotFound` if key is unknown. func (c *Cache) Get(key interface{}) func(*badger.Txn) (interface{}, error) { return func(tx *badger.Txn) (interface{}, error) { diff --git a/storage/badger/headers.go b/storage/badger/headers.go index 4e764c276a2..a7b9d8ed66a 100644 --- a/storage/badger/headers.go +++ b/storage/badger/headers.go @@ -107,6 +107,7 @@ func (h *Headers) retrieveTx(blockID flow.Identifier) func(*badger.Txn) (*flow.H } } +// results in `storage.ErrNotFound` for unknown height func (h *Headers) retrieveIdByHeightTx(height uint64) func(*badger.Txn) (flow.Identifier, error) { return func(tx *badger.Txn) (flow.Identifier, error) { blockID, err := h.heightCache.Get(height)(tx) @@ -138,6 +139,9 @@ func (h *Headers) ByHeight(height uint64) (*flow.Header, error) { return h.retrieveTx(blockID)(tx) } +// BlockIDByHeight the block ID that is finalized at the given height. It is an optimized version +// of `ByHeight` that skips retrieving the block. Expected errors during normal operations: +// - `storage.ErrNotFound` if no finalized block is known at given height. func (h *Headers) BlockIDByHeight(height uint64) (flow.Identifier, error) { tx := h.db.NewTransaction(false) defer tx.Discard() diff --git a/storage/badger/operation/dkg.go b/storage/badger/operation/dkg.go index 63b75166bd6..7a468ed9f36 100644 --- a/storage/badger/operation/dkg.go +++ b/storage/badger/operation/dkg.go @@ -15,6 +15,7 @@ import ( // CAUTION: This method stores confidential information and should only be // used in the context of the secrets database. This is enforced in the above // layer (see storage.DKGState). +// Error returns: storage.ErrAlreadyExists func InsertMyBeaconPrivateKey(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*badger.Txn) error { return insert(makePrefix(codeBeaconPrivateKey, epochCounter), info) } @@ -24,18 +25,21 @@ func InsertMyBeaconPrivateKey(epochCounter uint64, info *encodable.RandomBeaconP // CAUTION: This method stores confidential information and should only be // used in the context of the secrets database. This is enforced in the above // layer (see storage.DKGState). +// Error returns: storage.ErrNotFound func RetrieveMyBeaconPrivateKey(epochCounter uint64, info *encodable.RandomBeaconPrivKey) func(*badger.Txn) error { return retrieve(makePrefix(codeBeaconPrivateKey, epochCounter), info) } -// InsertDKGStartedForEpoch stores a flag indicating that the DKG has been started for -// the given epoch. +// InsertDKGStartedForEpoch stores a flag indicating that the DKG has been started for the given epoch. +// Returns: storage.ErrAlreadyExists +// Error returns: storage.ErrAlreadyExists func InsertDKGStartedForEpoch(epochCounter uint64) func(*badger.Txn) error { return insert(makePrefix(codeDKGStarted, epochCounter), true) } // RetrieveDKGStartedForEpoch retrieves the DKG started flag for the given epoch. -// If no flag is set, started is set to false. +// If no flag is set, started is set to false and no error is returned. +// No errors expected during normal operation. func RetrieveDKGStartedForEpoch(epochCounter uint64, started *bool) func(*badger.Txn) error { return func(tx *badger.Txn) error { err := retrieve(makePrefix(codeDKGStarted, epochCounter), started)(tx) @@ -53,11 +57,13 @@ func RetrieveDKGStartedForEpoch(epochCounter uint64, started *bool) func(*badger } // InsertDKGEndStateForEpoch stores the DKG end state for the epoch. +// Error returns: storage.ErrAlreadyExists func InsertDKGEndStateForEpoch(epochCounter uint64, endState flow.DKGEndState) func(*badger.Txn) error { return insert(makePrefix(codeDKGEnded, epochCounter), endState) } // RetrieveDKGEndStateForEpoch retrieves the DKG end state for the epoch. +// Error returns: storage.ErrNotFound func RetrieveDKGEndStateForEpoch(epochCounter uint64, endState *flow.DKGEndState) func(*badger.Txn) error { return retrieve(makePrefix(codeDKGEnded, epochCounter), endState) } diff --git a/storage/badger/operation/epoch.go b/storage/badger/operation/epoch.go index 37c377157ef..b5fcef7e029 100644 --- a/storage/badger/operation/epoch.go +++ b/storage/badger/operation/epoch.go @@ -35,10 +35,11 @@ func RetrieveEpochStatus(blockID flow.Identifier, status *flow.EpochStatus) func // SetEpochEmergencyFallbackTriggered sets a flag in the DB indicating that // epoch emergency fallback has been triggered, and the block where it was triggered. -// EECC can be triggered by 2 blocks: // -// 1. The first block of a new epoch, when that epoch has not been set up. -// 2. The block where an invalid service event is being applied to the state. +// EECC can be triggered in two ways: +// 1. Finalizing the first block past the epoch commitment deadline, when the +// next epoch has not yet been committed (see protocol.Params for more detail) +// 2. Finalizing a fork in which an invalid service event was incorporated. // // Calling this function multiple times is a no-op and returns no expected errors. func SetEpochEmergencyFallbackTriggered(blockID flow.Identifier) func(txn *badger.Txn) error { diff --git a/storage/badger/operation/headers.go b/storage/badger/operation/headers.go index bb429c69568..d8dfd7cb3f2 100644 --- a/storage/badger/operation/headers.go +++ b/storage/badger/operation/headers.go @@ -27,16 +27,6 @@ func LookupBlockHeight(height uint64, blockID *flow.Identifier) func(*badger.Txn return retrieve(makePrefix(codeHeightToBlock, height), blockID) } -// InsertBlockValidity marks a block as valid or invalid, defined by the consensus algorithm. -func InsertBlockValidity(blockID flow.Identifier, valid bool) func(*badger.Txn) error { - return insert(makePrefix(codeBlockValidity, blockID), valid) -} - -// RetrieveBlockValidity returns a block's validity wrt the consensus algorithm. -func RetrieveBlockValidity(blockID flow.Identifier, valid *bool) func(*badger.Txn) error { - return retrieve(makePrefix(codeBlockValidity, blockID), valid) -} - func InsertExecutedBlock(blockID flow.Identifier) func(*badger.Txn) error { return insert(makePrefix(codeExecutedBlock), blockID) } diff --git a/storage/badger/operation/interactions_test.go b/storage/badger/operation/interactions_test.go index d5c95c8968d..30fb6fd968f 100644 --- a/storage/badger/operation/interactions_test.go +++ b/storage/badger/operation/interactions_test.go @@ -17,25 +17,28 @@ import ( func TestStateInteractionsInsertCheckRetrieve(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - d1 := delta.NewView(func(owner, key string) (value flow.RegisterValue, err error) { - return nil, nil - }) + d1 := delta.NewDeltaView(nil) - d2 := delta.NewView(func(owner, key string) (value flow.RegisterValue, err error) { - return nil, nil - }) + d2 := delta.NewDeltaView(nil) + + id1 := flow.NewRegisterID( + string([]byte("\x89krg\u007fBN\x1d\xf5\xfb\xb8r\xbc4\xbd\x98ռ\xf1\xd0twU\xbf\x16N\xb4?,\xa0&;")), + "") + + id2 := flow.NewRegisterID(string([]byte{2}), "") + id3 := flow.NewRegisterID(string([]byte{3}), "") // some set and reads - err := d1.Set(string([]byte("\x89krg\u007fBN\x1d\xf5\xfb\xb8r\xbc4\xbd\x98ռ\xf1\xd0twU\xbf\x16N\xb4?,\xa0&;")), "", []byte("zażółć gęślą jaźń")) + err := d1.Set(id1, []byte("zażółć gęślą jaźń")) require.NoError(t, err) - err = d1.Set(string([]byte{2}), "", []byte("b")) + err = d1.Set(id2, []byte("b")) require.NoError(t, err) - err = d1.Set(string([]byte{2}), "", []byte("c")) + err = d1.Set(id2, []byte("c")) require.NoError(t, err) - _, err = d1.Get(string([]byte{2}), "") + _, err = d1.Get(id2) require.NoError(t, err) - _, err = d1.Get(string([]byte{3}), "") + _, err = d1.Get(id3) require.NoError(t, err) interactions := []*delta.Snapshot{&d1.Interactions().Snapshot, &d2.Interactions().Snapshot} diff --git a/storage/badger/operation/prefix.go b/storage/badger/operation/prefix.go index d1fefc5593c..400f475b2c6 100644 --- a/storage/badger/operation/prefix.go +++ b/storage/badger/operation/prefix.go @@ -16,20 +16,22 @@ const ( codeDBType = 2 // specifies a database type // codes for views with special meaning - codeStartedView = 10 // latest view hotstuff started - codeVotedView = 11 // latest view hotstuff voted on + codeSafetyData = 10 // safety data for hotstuff state + codeLivenessData = 11 // liveness data for hotstuff state // codes for fields associated with the root state - codeRootQuorumCertificate = 12 - codeSporkID = 13 - codeProtocolVersion = 14 + codeRootQuorumCertificate = 12 + codeSporkID = 13 + codeProtocolVersion = 14 + codeEpochCommitSafetyThreshold = 15 + codeSporkRootBlockHeight = 16 // code for heights with special meaning codeFinalizedHeight = 20 // latest finalized block height codeSealedHeight = 21 // latest sealed block height codeClusterHeight = 22 // latest finalized height on cluster codeExecutedBlock = 23 // latest executed block with max height - codeRootHeight = 24 // the height of the first loaded block + codeRootHeight = 24 // the height of the highest block contained in the root snapshot codeLastCompleteBlockHeight = 25 // the height of the last block for which all collections were received // codes for single entity storage diff --git a/storage/badger/operation/spork.go b/storage/badger/operation/spork.go index 3b67cc91d17..9f80afcddf9 100644 --- a/storage/badger/operation/spork.go +++ b/storage/badger/operation/spork.go @@ -18,6 +18,18 @@ func RetrieveSporkID(sporkID *flow.Identifier) func(*badger.Txn) error { return retrieve(makePrefix(codeSporkID), sporkID) } +// InsertSporkRootBlockHeight inserts the spork root block height for the present spork. +// A single database and protocol state instance spans at most one spork, so this is inserted +// exactly once, when bootstrapping the state. +func InsertSporkRootBlockHeight(height uint64) func(*badger.Txn) error { + return insert(makePrefix(codeSporkRootBlockHeight), height) +} + +// RetrieveSporkRootBlockHeight retrieves the spork root block height for the present spork. +func RetrieveSporkRootBlockHeight(height *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeSporkRootBlockHeight), height) +} + // InsertProtocolVersion inserts the protocol version for the present spork. // A single database and protocol state instance spans at most one spork, and // a spork has exactly one protocol version for its duration, so this is @@ -30,3 +42,18 @@ func InsertProtocolVersion(version uint) func(*badger.Txn) error { func RetrieveProtocolVersion(version *uint) func(*badger.Txn) error { return retrieve(makePrefix(codeProtocolVersion), version) } + +// InsertEpochCommitSafetyThreshold inserts the epoch commit safety threshold +// for the present spork. +// A single database and protocol state instance spans at most one spork, and +// a spork has exactly one protocol version for its duration, so this is +// inserted exactly once, when bootstrapping the state. +func InsertEpochCommitSafetyThreshold(threshold uint64) func(*badger.Txn) error { + return insert(makePrefix(codeEpochCommitSafetyThreshold), threshold) +} + +// RetrieveEpochCommitSafetyThreshold retrieves the epoch commit safety threshold +// for the present spork. +func RetrieveEpochCommitSafetyThreshold(threshold *uint64) func(*badger.Txn) error { + return retrieve(makePrefix(codeEpochCommitSafetyThreshold), threshold) +} diff --git a/storage/badger/operation/spork_test.go b/storage/badger/operation/spork_test.go index e3ccd7ca9da..a000df60561 100644 --- a/storage/badger/operation/spork_test.go +++ b/storage/badger/operation/spork_test.go @@ -41,3 +41,20 @@ func TestProtocolVersion_InsertRetrieve(t *testing.T) { assert.Equal(t, version, actual) }) } + +// TestEpochCommitSafetyThreshold_InsertRetrieve tests that we can insert and +// retrieve epoch commit safety threshold values. +func TestEpochCommitSafetyThreshold_InsertRetrieve(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + threshold := rand.Uint64() + + err := db.Update(InsertEpochCommitSafetyThreshold(threshold)) + require.NoError(t, err) + + var actual uint64 + err = db.View(RetrieveEpochCommitSafetyThreshold(&actual)) + require.NoError(t, err) + + assert.Equal(t, threshold, actual) + }) +} diff --git a/storage/badger/operation/views.go b/storage/badger/operation/views.go index 62df83244d1..21f31316f1f 100644 --- a/storage/badger/operation/views.go +++ b/storage/badger/operation/views.go @@ -3,35 +3,36 @@ package operation import ( "github.com/dgraph-io/badger/v2" + "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/model/flow" ) -// InsertStartedView inserts a view into the database. -func InsertStartedView(chainID flow.ChainID, view uint64) func(*badger.Txn) error { - return insert(makePrefix(codeStartedView, chainID), view) +// InsertSafetyData inserts safety data into the database. +func InsertSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return insert(makePrefix(codeSafetyData, chainID), safetyData) } -// UpdateStartedView updates the view in the database. -func UpdateStartedView(chainID flow.ChainID, view uint64) func(*badger.Txn) error { - return update(makePrefix(codeStartedView, chainID), view) +// UpdateSafetyData updates safety data in the database. +func UpdateSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return update(makePrefix(codeSafetyData, chainID), safetyData) } -// RetrieveStartedView retrieves a view from the database. -func RetrieveStartedView(chainID flow.ChainID, view *uint64) func(*badger.Txn) error { - return retrieve(makePrefix(codeStartedView, chainID), view) +// RetrieveSafetyData retrieves safety data from the database. +func RetrieveSafetyData(chainID flow.ChainID, safetyData *hotstuff.SafetyData) func(*badger.Txn) error { + return retrieve(makePrefix(codeSafetyData, chainID), safetyData) } -// InsertVotedView inserts a view into the database. -func InsertVotedView(chainID flow.ChainID, view uint64) func(*badger.Txn) error { - return insert(makePrefix(codeVotedView, chainID), view) +// InsertLivenessData inserts liveness data into the database. +func InsertLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return insert(makePrefix(codeLivenessData, chainID), livenessData) } -// UpdateVotedView updates the view in the database. -func UpdateVotedView(chainID flow.ChainID, view uint64) func(*badger.Txn) error { - return update(makePrefix(codeVotedView, chainID), view) +// UpdateLivenessData updates liveness data in the database. +func UpdateLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return update(makePrefix(codeLivenessData, chainID), livenessData) } -// RetrieveVotedView retrieves a view from the database. -func RetrieveVotedView(chainID flow.ChainID, view *uint64) func(*badger.Txn) error { - return retrieve(makePrefix(codeVotedView, chainID), view) +// RetrieveLivenessData retrieves liveness data from the database. +func RetrieveLivenessData(chainID flow.ChainID, livenessData *hotstuff.LivenessData) func(*badger.Txn) error { + return retrieve(makePrefix(codeLivenessData, chainID), livenessData) } diff --git a/storage/badger/seals.go b/storage/badger/seals.go index 02a8b11fdbe..aa68511ed7e 100644 --- a/storage/badger/seals.go +++ b/storage/badger/seals.go @@ -91,7 +91,7 @@ func (s *Seals) FinalizedSealForBlock(blockID flow.Identifier) (*flow.Seal, erro var sealID flow.Identifier err := s.db.View(operation.LookupBySealedBlockID(blockID, &sealID)) if err != nil { - return nil, fmt.Errorf("failed to retrieve seal for fork with head %x: %w", blockID, err) + return nil, fmt.Errorf("failed to retrieve seal for block %x: %w", blockID, err) } return s.ByID(sealID) } diff --git a/storage/dkg.go b/storage/dkg.go index 26da4a4ef0e..fdd749ae7ae 100644 --- a/storage/dkg.go +++ b/storage/dkg.go @@ -11,15 +11,19 @@ import ( type DKGState interface { // SetDKGStarted sets the flag indicating the DKG has started for the given epoch. + // Error returns: storage.ErrAlreadyExists SetDKGStarted(epochCounter uint64) error // GetDKGStarted checks whether the DKG has been started for the given epoch. + // No errors expected during normal operation. GetDKGStarted(epochCounter uint64) (bool, error) // SetDKGEndState stores that the DKG has ended, and its end state. + // Error returns: storage.ErrAlreadyExists SetDKGEndState(epochCounter uint64, endState flow.DKGEndState) error // GetDKGEndState retrieves the end state for the given DKG. + // Error returns: storage.ErrNotFound GetDKGEndState(epochCounter uint64) (flow.DKGEndState, error) // InsertMyBeaconPrivateKey stores the random beacon private key for an epoch. @@ -27,6 +31,7 @@ type DKGState interface { // CAUTION: these keys are stored before they are validated against the // canonical key vector and may not be valid for use in signing. Use SafeBeaconKeys // to guarantee only keys safe for signing are returned + // Error returns: storage.ErrAlreadyExists InsertMyBeaconPrivateKey(epochCounter uint64, key crypto.PrivateKey) error // RetrieveMyBeaconPrivateKey retrieves the random beacon private key for an epoch. @@ -34,6 +39,7 @@ type DKGState interface { // CAUTION: these keys are stored before they are validated against the // canonical key vector and may not be valid for use in signing. Use SafeBeaconKeys // to guarantee only keys safe for signing are returned + // Error returns: storage.ErrNotFound RetrieveMyBeaconPrivateKey(epochCounter uint64) (crypto.PrivateKey, error) } @@ -47,5 +53,6 @@ type SafeBeaconKeys interface { // * (key, true, nil) if the key is present and confirmed valid // * (nil, false, nil) if no key was generated or the key has been marked invalid (by SetDKGEnded) // * (nil, false, error) for any other condition, or exception + // Error returns: storage.ErrNotFound RetrieveMyBeaconPrivateKey(epochCounter uint64) (key crypto.PrivateKey, safe bool, err error) } diff --git a/storage/headers.go b/storage/headers.go index 0abe6274a37..9bf7ee9d15a 100644 --- a/storage/headers.go +++ b/storage/headers.go @@ -12,14 +12,19 @@ type Headers interface { // Store will store a header. Store(header *flow.Header) error - // ByBlockID returns the header with the given ID. It is available for - // finalized and ambiguous blocks. + // ByBlockID returns the header with the given ID. It is available for finalized and ambiguous blocks. + // Error returns: + // - ErrNotFound if no block header with the given ID exists ByBlockID(blockID flow.Identifier) (*flow.Header, error) - // ByHeight returns the block with the given number. It is only available - // for finalized blocks. + // ByHeight returns the block with the given number. It is only available for finalized blocks. ByHeight(height uint64) (*flow.Header, error) + // BlockIDByHeight the block ID that is finalized at the given height. It is an optimized version + // of `ByHeight` that skips retrieving the block. Expected errors during normal operations: + // * `storage.ErrNotFound` if no finalized block is known at given height + BlockIDByHeight(height uint64) (flow.Identifier, error) + // ByParentID finds all children for the given parent block. The returned headers // might be unfinalized; if there is more than one, at least one of them has to // be unfinalized. diff --git a/storage/ledger.go b/storage/ledger.go index f79af359884..e76b4f2b9f3 100755 --- a/storage/ledger.go +++ b/storage/ledger.go @@ -2,13 +2,11 @@ package storage import ( "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" ) // Ledger takes care of storing registers (key, value pairs) providing proof of correctness // we aim to store a state of the order of 10^10 registers with up to 1M historic state versions type Ledger interface { - module.ReadyDoneAware EmptyStateCommitment() flow.StateCommitment // Trusted methods (without proof) diff --git a/storage/mock/headers.go b/storage/mock/headers.go index 2e0ede53e90..990efe3a7e3 100644 --- a/storage/mock/headers.go +++ b/storage/mock/headers.go @@ -42,6 +42,29 @@ func (_m *Headers) BatchRemoveChunkBlockIndexByChunkID(chunkID flow.Identifier, return r0 } +// BlockIDByHeight provides a mock function with given fields: height +func (_m *Headers) BlockIDByHeight(height uint64) (flow.Identifier, error) { + ret := _m.Called(height) + + var r0 flow.Identifier + if rf, ok := ret.Get(0).(func(uint64) flow.Identifier); ok { + r0 = rf(height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(flow.Identifier) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(uint64) error); ok { + r1 = rf(height) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ByBlockID provides a mock function with given fields: blockID func (_m *Headers) ByBlockID(blockID flow.Identifier) (*flow.Header, error) { ret := _m.Called(blockID) diff --git a/storage/mock/ledger.go b/storage/mock/ledger.go index 92d2bf073ba..c20181a7d3e 100644 --- a/storage/mock/ledger.go +++ b/storage/mock/ledger.go @@ -12,22 +12,6 @@ type Ledger struct { mock.Mock } -// Done provides a mock function with given fields: -func (_m *Ledger) Done() <-chan struct{} { - ret := _m.Called() - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - // EmptyStateCommitment provides a mock function with given fields: func (_m *Ledger) EmptyStateCommitment() flow.StateCommitment { ret := _m.Called() @@ -99,22 +83,6 @@ func (_m *Ledger) GetRegistersWithProof(registerIDs []flow.RegisterID, stateComm return r0, r1, r2 } -// Ready provides a mock function with given fields: -func (_m *Ledger) Ready() <-chan struct{} { - ret := _m.Called() - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - // UpdateRegisters provides a mock function with given fields: registerIDs, values, stateCommitment func (_m *Ledger) UpdateRegisters(registerIDs []flow.RegisterID, values [][]byte, stateCommitment flow.StateCommitment) (flow.StateCommitment, error) { ret := _m.Called(registerIDs, values, stateCommitment) diff --git a/storage/mocks/storage.go b/storage/mocks/storage.go index d7397f4c546..7e5fd6d46bd 100644 --- a/storage/mocks/storage.go +++ b/storage/mocks/storage.go @@ -216,6 +216,21 @@ func (mr *MockHeadersMockRecorder) BatchRemoveChunkBlockIndexByChunkID(arg0, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchRemoveChunkBlockIndexByChunkID", reflect.TypeOf((*MockHeaders)(nil).BatchRemoveChunkBlockIndexByChunkID), arg0, arg1) } +// BlockIDByHeight mocks base method +func (m *MockHeaders) BlockIDByHeight(arg0 uint64) (flow.Identifier, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BlockIDByHeight", arg0) + ret0, _ := ret[0].(flow.Identifier) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BlockIDByHeight indicates an expected call of BlockIDByHeight +func (mr *MockHeadersMockRecorder) BlockIDByHeight(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlockIDByHeight", reflect.TypeOf((*MockHeaders)(nil).BlockIDByHeight), arg0) +} + // ByBlockID mocks base method func (m *MockHeaders) ByBlockID(arg0 flow.Identifier) (*flow.Header, error) { m.ctrl.T.Helper() diff --git a/utils/debug/remoteView.go b/utils/debug/remoteView.go index 2b5eb69c73d..b63e77b1e6f 100644 --- a/utils/debug/remoteView.go +++ b/utils/debug/remoteView.go @@ -5,6 +5,7 @@ import ( "fmt" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow/protobuf/go/flow/execution" @@ -54,7 +55,7 @@ func WithBlockID(blockID flow.Identifier) RemoteViewOption { } func NewRemoteView(grpcAddress string, opts ...RemoteViewOption) *RemoteView { - conn, err := grpc.Dial(grpcAddress, grpc.WithInsecure()) //nolint:staticcheck + conn, err := grpc.Dial(grpcAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { panic(err) } @@ -146,35 +147,35 @@ func (v *RemoteView) DropDelta() { v.Delta = make(map[string]flow.RegisterValue) } -func (v *RemoteView) Set(owner, key string, value flow.RegisterValue) error { - v.Delta[owner+"~"+key] = value +func (v *RemoteView) Set(id flow.RegisterID, value flow.RegisterValue) error { + v.Delta[id.Owner+"~"+id.Key] = value return nil } -func (v *RemoteView) Get(owner, key string) (flow.RegisterValue, error) { +func (v *RemoteView) Get(id flow.RegisterID) (flow.RegisterValue, error) { // first check the delta - value, found := v.Delta[owner+"~"+key] + value, found := v.Delta[id.Owner+"~"+id.Key] if found { return value, nil } // then check the read cache - value, found = v.Cache.Get(owner, key) + value, found = v.Cache.Get(id.Owner, id.Key) if found { return value, nil } // then call the parent (if exist) if v.Parent != nil { - return v.Parent.Get(owner, key) + return v.Parent.Get(id) } // last use the grpc api the req := &execution.GetRegisterAtBlockIDRequest{ BlockId: []byte(v.BlockID), - RegisterOwner: []byte(owner), - RegisterKey: []byte(key), + RegisterOwner: []byte(id.Owner), + RegisterKey: []byte(id.Key), } // TODO use a proper context for timeouts @@ -183,28 +184,23 @@ func (v *RemoteView) Get(owner, key string) (flow.RegisterValue, error) { return nil, err } - v.Cache.Set(owner, key, resp.Value) + v.Cache.Set(id.Owner, id.Key, resp.Value) // append value to the file cache return resp.Value, nil } -// returns all the registers that has been touched -func (v *RemoteView) AllRegisters() []flow.RegisterID { +// returns all the register ids that has been updated +func (v *RemoteView) UpdatedRegisterIDs() []flow.RegisterID { panic("Not implemented yet") } -func (v *RemoteView) UpdatedRegisters() flow.RegisterEntries { +// returns all the register ids that has been touched +func (v *RemoteView) AllRegisterIDs() []flow.RegisterID { panic("Not implemented yet") } -func (v *RemoteView) Touch(owner, key string) error { - // no-op for now - return nil -} - -func (v *RemoteView) Delete(owner, key string) error { - v.Delta[owner+"~"+key] = nil - return nil +func (v *RemoteView) UpdatedRegisters() flow.RegisterEntries { + panic("Not implemented yet") } diff --git a/utils/grpcutils/grpc.go b/utils/grpcutils/grpc.go index 15c2dbd1bb2..3167b025dff 100644 --- a/utils/grpcutils/grpc.go +++ b/utils/grpcutils/grpc.go @@ -6,21 +6,40 @@ import ( "encoding/hex" "fmt" - libp2ptls "github.com/libp2p/go-libp2p-tls" lcrypto "github.com/libp2p/go-libp2p/core/crypto" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/network/p2p/keyutils" ) -// DefaultMaxMsgSize use 16MB as the default message size limit. +// DefaultMaxMsgSize use 20MB as the default message size limit. // grpc library default is 4MB -const DefaultMaxMsgSize = 1024 * 1024 * 16 +const DefaultMaxMsgSize = 1024 * 1024 * 20 + +// CertificateConfig is used to configure an Certificate +type CertificateConfig struct { + opts []libp2ptls.IdentityOption +} + +// CertificateOption transforms an CertificateConfig to apply optional settings. +type CertificateOption func(r *CertificateConfig) + +// WithCertTemplate specifies the template to use when generating a new certificate. +func WithCertTemplate(template *x509.Certificate) CertificateOption { + return func(c *CertificateConfig) { + c.opts = append(c.opts, libp2ptls.WithCertTemplate(template)) + } +} // X509Certificate generates a self-signed x509 TLS certificate from the given key. The generated certificate // includes a libp2p extension that specifies the public key and the signature. The certificate does not include any // SAN extension. -func X509Certificate(privKey crypto.PrivateKey) (*tls.Certificate, error) { +func X509Certificate(privKey crypto.PrivateKey, opts ...CertificateOption) (*tls.Certificate, error) { + config := CertificateConfig{} + for _, opt := range opts { + opt(&config) + } // convert the Flow crypto private key to a Libp2p private crypto key libP2PKey, err := keyutils.LibP2PPrivKeyFromFlow(privKey) @@ -29,7 +48,7 @@ func X509Certificate(privKey crypto.PrivateKey) (*tls.Certificate, error) { } // create a libp2p Identity from the libp2p private key - id, err := libp2ptls.NewIdentity(libP2PKey) + id, err := libp2ptls.NewIdentity(libP2PKey, config.opts...) if err != nil { return nil, fmt.Errorf("could not generate identity: %w", err) } @@ -49,6 +68,9 @@ func X509Certificate(privKey crypto.PrivateKey) (*tls.Certificate, error) { // DefaultServerTLSConfig returns the default TLS server config with the given cert for a secure GRPC server func DefaultServerTLSConfig(cert *tls.Certificate) *tls.Config { + + // TODO(rbtz): remove after we pick up https://github.com/securego/gosec/pull/903 + // #nosec G402 tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{*cert}, diff --git a/utils/grpcutils/grpc_test.go b/utils/grpcutils/grpc_test.go index 91fa11eefdf..3fe19ea38b9 100644 --- a/utils/grpcutils/grpc_test.go +++ b/utils/grpcutils/grpc_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - libp2ptls "github.com/libp2p/go-libp2p-tls" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/network/p2p/keyutils" diff --git a/utils/unittest/epoch_builder.go b/utils/unittest/epoch_builder.go index fe9e5f8e39c..95666ad96e9 100644 --- a/utils/unittest/epoch_builder.go +++ b/utils/unittest/epoch_builder.go @@ -377,9 +377,6 @@ func (builder *EpochBuilder) addBlock(block *flow.Block) { err := state.Extend(context.Background(), block) require.NoError(builder.t, err) - err = state.MarkValid(blockID) - require.NoError(builder.t, err) - err = state.Finalize(context.Background(), blockID) require.NoError(builder.t, err) } diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 316b48bb67a..88923795b63 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -24,7 +24,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "48bc690ac7ebf7cdb41df24c6f8ba011b206408623b6d56b3723910664cdb5aa" +const GenesisStateCommitmentHex = "c10e91622a92413fac93998daa5c4515ce9815c764ddc94b148994f9797ca705" var GenesisStateCommitment flow.StateCommitment diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 898a8868959..8b558403bc4 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/cadence" sdk "github.com/onflow/flow-go-sdk" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" @@ -34,6 +35,7 @@ import ( "github.com/onflow/flow-go/model/verification" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/mempool/entity" + "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/module/updatable_configs" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" @@ -66,6 +68,11 @@ func RandomAddressFixture() flow.Address { return addr } +// Uint64InRange returns a uint64 value drawn from the uniform random distribution [min,max]. +func Uint64InRange(min, max uint64) uint64 { + return min + uint64(rand.Intn(int(max)+1-int(min))) +} + func RandomSDKAddressFixture() sdk.Address { addr := RandomAddressFixture() var sdkAddr sdk.Address @@ -325,16 +332,22 @@ func WithoutGuarantee(payload *flow.Payload) { } func StateInteractionsFixture() *delta.Snapshot { - return &delta.NewView(nil).Interactions().Snapshot + return &delta.NewDeltaView(nil).Interactions().Snapshot } -func BlockWithParentAndProposerFixture(parent *flow.Header, proposer flow.Identifier, participantCount int) flow.Block { +func BlockWithParentAndProposerFixture(t *testing.T, parent *flow.Header, proposer flow.Identifier) flow.Block { block := BlockWithParentFixture(parent) + indices, err := signature.EncodeSignersToIndices( + []flow.Identifier{proposer}, []flow.Identifier{proposer}) + require.NoError(t, err) + block.Header.ProposerID = proposer - indices := bitutils.MakeBitVector(10) - bitutils.SetBit(indices, 1) block.Header.ParentVoterIndices = indices + if block.Header.LastViewTC != nil { + block.Header.LastViewTC.SignerIndices = indices + block.Header.LastViewTC.NewestQC.SignerIndices = indices + } return *block } @@ -438,6 +451,19 @@ func BlockHeaderFixtureOnChain(chainID flow.ChainID, opts ...func(header *flow.H func BlockHeaderWithParentFixture(parent *flow.Header) *flow.Header { height := parent.Height + 1 view := parent.View + 1 + uint64(rand.Intn(10)) // Intn returns [0, n) + var lastViewTC *flow.TimeoutCertificate + if view != parent.View+1 { + newestQC := QuorumCertificateFixture(func(qc *flow.QuorumCertificate) { + qc.View = parent.View + }) + lastViewTC = &flow.TimeoutCertificate{ + View: view - 1, + NewestQCViews: []uint64{newestQC.View}, + NewestQC: newestQC, + SignerIndices: SignerIndicesFixture(4), + SigData: SignatureFixture(), + } + } return &flow.Header{ ChainID: parent.ChainID, ParentID: parent.ID(), @@ -445,10 +471,12 @@ func BlockHeaderWithParentFixture(parent *flow.Header) *flow.Header { PayloadHash: IdentifierFixture(), Timestamp: time.Now().UTC(), View: view, + ParentView: parent.View, ParentVoterIndices: SignerIndicesFixture(4), ParentVoterSigData: QCSigDataFixture(), ProposerID: IdentifierFixture(), ProposerSigData: SignatureFixture(), + LastViewTC: lastViewTC, } } @@ -486,6 +514,7 @@ func ClusterBlockWithParent(parent *cluster.Block) cluster.Block { header.ChainID = parent.Header.ChainID header.Timestamp = time.Now() header.ParentID = parent.ID() + header.ParentView = parent.Header.View header.PayloadHash = payload.Hash() block := cluster.Block{ @@ -992,7 +1021,7 @@ func IdentityFixture(opts ...func(*flow.Identity)) *flow.Identity { stakingKey := StakingPrivKeyByIdentifier(nodeID) identity := flow.Identity{ NodeID: nodeID, - Address: fmt.Sprintf("address-%x:0", nodeID[0:7]), + Address: fmt.Sprintf("address-%x", nodeID[0:7]), Role: flow.RoleConsensus, Weight: 1000, StakingPubKey: stakingKey.PublicKey(), @@ -1695,6 +1724,13 @@ func QCWithSignerIndices(signerIndices []byte) func(*flow.QuorumCertificate) { } } +func QCWithRootBlockID(blockID flow.Identifier) func(*flow.QuorumCertificate) { + return func(qc *flow.QuorumCertificate) { + qc.BlockID = blockID + qc.View = 0 + } +} + func VoteFixture(opts ...func(vote *hotstuff.Vote)) *hotstuff.Vote { vote := &hotstuff.Vote{ View: uint64(rand.Uint32()), @@ -1908,7 +1944,7 @@ func BootstrapFixture(participants flow.IdentityList, opts ...func(*flow.Block)) // example one as returned from BootstrapFixture. func RootSnapshotFixture(participants flow.IdentityList, opts ...func(*flow.Block)) *inmem.Snapshot { block, result, seal := BootstrapFixture(participants.Sort(order.Canonical), opts...) - qc := QuorumCertificateFixture(QCWithBlockID(block.ID())) + qc := QuorumCertificateFixture(QCWithRootBlockID(block.ID())) root, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc) if err != nil { panic(err) diff --git a/utils/unittest/irrecoverable.go b/utils/unittest/irrecoverable.go new file mode 100644 index 00000000000..582bc82adc2 --- /dev/null +++ b/utils/unittest/irrecoverable.go @@ -0,0 +1,19 @@ +package unittest + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// FailOnIrrecoverableError waits for either the done signal, or an error +// to be sent over the error channel. If an error is observed, it is logged +// and the test is failed. +// Must be invoked as a goroutine. +func FailOnIrrecoverableError(t *testing.T, done <-chan struct{}, errCh <-chan error) { + select { + case <-done: + case err := <-errCh: + assert.NoError(t, err, "observed unexpected irrecoverable error") + } +} diff --git a/utils/unittest/mocks/epoch_query.go b/utils/unittest/mocks/epoch_query.go index 02b450e23e4..a624a655dd7 100644 --- a/utils/unittest/mocks/epoch_query.go +++ b/utils/unittest/mocks/epoch_query.go @@ -1,17 +1,20 @@ package mocks import ( + "sync" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/invalid" ) // EpochQuery implements protocol.EpochQuery for testing purposes. +// Safe for concurrent use by multiple goroutines. type EpochQuery struct { t *testing.T + mu sync.RWMutex counter uint64 // represents the current epoch byCounter map[uint64]protocol.Epoch // all epochs } @@ -31,10 +34,14 @@ func NewEpochQuery(t *testing.T, counter uint64, epochs ...protocol.Epoch) *Epoc } func (mock *EpochQuery) Current() protocol.Epoch { + mock.mu.RLock() + defer mock.mu.RUnlock() return mock.byCounter[mock.counter] } func (mock *EpochQuery) Next() protocol.Epoch { + mock.mu.RLock() + defer mock.mu.RUnlock() epoch, exists := mock.byCounter[mock.counter+1] if !exists { return invalid.NewEpoch(protocol.ErrNextEpochNotSetup) @@ -43,6 +50,8 @@ func (mock *EpochQuery) Next() protocol.Epoch { } func (mock *EpochQuery) Previous() protocol.Epoch { + mock.mu.RLock() + defer mock.mu.RUnlock() epoch, exists := mock.byCounter[mock.counter-1] if !exists { return invalid.NewEpoch(protocol.ErrNoPreviousEpoch) @@ -51,15 +60,21 @@ func (mock *EpochQuery) Previous() protocol.Epoch { } func (mock *EpochQuery) ByCounter(counter uint64) protocol.Epoch { + mock.mu.RLock() + defer mock.mu.RUnlock() return mock.byCounter[counter] } func (mock *EpochQuery) Transition() { + mock.mu.Lock() + defer mock.mu.Unlock() mock.counter++ } func (mock *EpochQuery) Add(epoch protocol.Epoch) { + mock.mu.Lock() + defer mock.mu.Unlock() counter, err := epoch.Counter() - assert.Nil(mock.t, err, "cannot add epoch with invalid counter") + require.NoError(mock.t, err, "cannot add epoch with invalid counter") mock.byCounter[counter] = epoch } diff --git a/utils/unittest/mocks/execution_state.go b/utils/unittest/mocks/execution_state.go deleted file mode 100644 index 75f593957ca..00000000000 --- a/utils/unittest/mocks/execution_state.go +++ /dev/null @@ -1,61 +0,0 @@ -package mocks - -import ( - "context" - "sync" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/engine/execution/state" - - "github.com/onflow/flow-go/engine/execution/state/mock" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/utils/unittest" -) - -// ExecutionState is a mocked version of execution state that -// simulates some of its behavior for testing purpose -type ExecutionState struct { - sync.Mutex - mock.ExecutionState - commits map[flow.Identifier]flow.StateCommitment -} - -func NewExecutionState(seal *flow.Seal) *ExecutionState { - commits := make(map[flow.Identifier]flow.StateCommitment) - commits[seal.BlockID] = seal.FinalState - return &ExecutionState{ - commits: commits, - } -} - -func (es *ExecutionState) PersistStateCommitment(ctx context.Context, blockID flow.Identifier, commit flow.StateCommitment) error { - es.Lock() - defer es.Unlock() - es.commits[blockID] = commit - return nil -} - -func (es *ExecutionState) StateCommitmentByBlockID(ctx context.Context, blockID flow.Identifier) (flow.StateCommitment, error) { - es.Lock() - defer es.Unlock() - commit, ok := es.commits[blockID] - if !ok { - return flow.DummyStateCommitment, storage.ErrNotFound - } - - return commit, nil -} - -func (es *ExecutionState) ExecuteBlock(t *testing.T, block *flow.Block) { - parentExecuted, err := state.IsBlockExecuted(context.Background(), es, block.Header.ParentID) - require.NoError(t, err) - require.True(t, parentExecuted, "parent block not executed") - require.NoError(t, - es.PersistStateCommitment( - context.Background(), - block.ID(), - unittest.StateCommitmentFixture())) -} diff --git a/utils/unittest/mocks/protocol_state.go b/utils/unittest/mocks/protocol_state.go index dcd82ede464..7eef54109fb 100644 --- a/utils/unittest/mocks/protocol_state.go +++ b/utils/unittest/mocks/protocol_state.go @@ -50,10 +50,22 @@ func (p *Params) SporkID() (flow.Identifier, error) { return flow.ZeroID, fmt.Errorf("not implemented") } +func (p *Params) SporkRootBlockHeight() (uint64, error) { + return 0, fmt.Errorf("not implemented") +} + func (p *Params) ProtocolVersion() (uint, error) { return 0, fmt.Errorf("not implemented") } +func (p *Params) EpochCommitSafetyThreshold() (uint64, error) { + return 0, fmt.Errorf("not implemented") +} + +func (p *Params) EpochFallbackTriggered() (bool, error) { + return false, fmt.Errorf("not implemented") +} + func (p *Params) Root() (*flow.Header, error) { return p.state.root.Header, nil } @@ -115,12 +127,6 @@ func (ps *ProtocolState) Final() protocol.Snapshot { mocked.ReturnArguments = mock.Arguments{pendings, nil} } - mocked = snapshot.On("ValidDescendants") - mocked.RunFn = func(args mock.Arguments) { - // not concurrent safe - pendings := pending(ps, finalID) - mocked.ReturnArguments = mock.Arguments{pendings, nil} - } return snapshot } diff --git a/utils/unittest/protocol_state.go b/utils/unittest/protocol_state.go index bf7e4361a8d..9253d2baef2 100644 --- a/utils/unittest/protocol_state.go +++ b/utils/unittest/protocol_state.go @@ -69,10 +69,11 @@ func FinalizedProtocolStateWithParticipants(participants flow.IdentityList) ( return block, snapshot, state, sealedSnapshot } -// SealBlock seals a block by building two blocks on it, the first containing -// a receipt for the block, the second containing a seal for the block. -// Returns the block containing the seal. -func SealBlock(t *testing.T, st protocol.MutableState, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) *flow.Header { +// SealBlock seals a block B by building two blocks on it, the first containing +// a receipt for the block (BR), the second (BS) containing a seal for the block. +// B <- BR(Result_B) <- BS(Seal_B) +// Returns the two generated blocks. +func SealBlock(t *testing.T, st protocol.MutableState, block *flow.Block, receipt *flow.ExecutionReceipt, seal *flow.Seal) (br *flow.Header, bs *flow.Header) { block2 := BlockWithParentFixture(block.Header) block2.SetPayload(flow.Payload{ @@ -89,5 +90,13 @@ func SealBlock(t *testing.T, st protocol.MutableState, block *flow.Block, receip err = st.Extend(context.Background(), block3) require.NoError(t, err) - return block3.Header + return block2.Header, block3.Header +} + +// InsertAndFinalize inserts, then finalizes, the input block. +func InsertAndFinalize(t *testing.T, st protocol.MutableState, block *flow.Block) { + err := st.Extend(context.Background(), block) + require.NoError(t, err) + err = st.Finalize(context.Background(), block.ID()) + require.NoError(t, err) } diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index 89afff7b75a..6c693d678a9 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -135,7 +135,7 @@ func ExpectPanic(expectedMsg string, t *testing.T) { // AssertReturnsBefore asserts that the given function returns before the // duration expires. -func AssertReturnsBefore(t *testing.T, f func(), duration time.Duration, msgAndArgs ...interface{}) { +func AssertReturnsBefore(t *testing.T, f func(), duration time.Duration, msgAndArgs ...interface{}) bool { done := make(chan struct{}) go func() { @@ -148,8 +148,16 @@ func AssertReturnsBefore(t *testing.T, f func(), duration time.Duration, msgAndA t.Log("function did not return in time") assert.Fail(t, "function did not close in time", msgAndArgs...) case <-done: - return + return true } + return false +} + +// ClosedChannel returns a closed channel. +func ClosedChannel() <-chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch } // AssertClosesBefore asserts that the given channel closes before the